{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[WARNING  | root               ]: Supported flash-attn versions are >= 2.1.1, <= 2.6.3. Found flash-attn 2.7.4.post1.\n",
      "/sharedata/mdy/miniforge/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import triton\n",
    "import math\n",
    "from copy import deepcopy\n",
    "import os\n",
    "os.environ['TRITON_PRINT_AUTOTUNING'] = '1'\n",
    "from dyt import DYT\n",
    "from dyt2 import DYT as DYT2\n",
    "from rmsnorm import TritonRMSNorm\n",
    "from apex.normalization import FusedRMSNorm\n",
    "from transformer_engine.pytorch.module.rmsnorm import RMSNorm as TERMSNorm\n",
    "from liger_kernel.transformers.dyt import LigerDyT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "dtype = torch.bfloat16\n",
    "device = 'cuda'\n",
    "bs, seq_len, dim = 1, 2048, 8192\n",
    "beta = True\n",
    "x1 = torch.randn(bs, seq_len, dim, device=device, dtype=dtype)\n",
    "x1.requires_grad_(True)\n",
    "\n",
    "x2 = deepcopy(x1)\n",
    "dyt1 = DYT2(dim, beta=beta).to(device).to(dtype)\n",
    "dyt1.gemma.data.copy_(torch.randn(dim))\n",
    "if beta:\n",
    "    dyt1.beta.data.copy_(torch.randn(dim))\n",
    "dyt2 = deepcopy(dyt1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Triton autotuning for function _dyt_bwd_kernel finished after 21.53s; best config selected: BLOCK_N: 1024, num_warps: 4, num_ctas: 1, num_stages: 4, maxnreg: None;\n",
      "tensor(0.0312, device='cuda:0', dtype=torch.bfloat16, grad_fn=<MaxBackward1>) tensor(0.0007, device='cuda:0', dtype=torch.bfloat16, grad_fn=<MeanBackward0>)\n",
      "tensor(0.0312, device='cuda:0', dtype=torch.bfloat16) tensor(0.0005, device='cuda:0', dtype=torch.bfloat16)\n",
      "tensor(0.5000, device='cuda:0', dtype=torch.bfloat16) tensor(0.0342, device='cuda:0', dtype=torch.bfloat16)\n",
      "tensor(0., device='cuda:0', dtype=torch.bfloat16) tensor(0., device='cuda:0', dtype=torch.bfloat16)\n",
      "tensor([0.], device='cuda:0', dtype=torch.bfloat16)\n"
     ]
    }
   ],
   "source": [
    "y1 = dyt1(x1, \"torch\")\n",
    "y2 = dyt2(x2, \"triton\")\n",
    "dy = torch.randn_like(y1)\n",
    "y1.backward(dy)\n",
    "y2.backward(dy)\n",
    "print((y1-y2).abs().max(), (y1-y2).abs().mean())\n",
    "print((x1.grad-x2.grad).abs().max(), (x1.grad-x2.grad).abs().mean())\n",
    "print((dyt1.gemma.grad-dyt2.gemma.grad).abs().max(), (dyt1.gemma.grad-dyt2.gemma.grad).abs().mean())\n",
    "if beta:\n",
    "    print((dyt1.beta.grad-dyt2.beta.grad).abs().max(), (dyt1.beta.grad-dyt2.beta.grad).abs().mean())\n",
    "print((dyt1.alpha.grad-dyt2.alpha.grad).abs() / dyt1.alpha.grad.abs())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkAAAAGwCAYAAABB4NqyAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjEsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvc2/+5QAAAAlwSFlzAAAPYQAAD2EBqD+naQAAdDhJREFUeJzt3Xt8zvX/x/HHtfPJ5jA7NkZEOU3G6ES1jERKkYpZvkpFtMohGUU5JEnkUDlUv0p9K6l8SctUiJyS0hxz3mxkY7Pj9fn9Mbu4mPO2zw7P++32ue26Ptf787le7+G6nt6f9+fzsRiGYSAiIiJSiTiYXYCIiIhIaVMAEhERkUpHAUhEREQqHQUgERERqXQUgERERKTSUQASERGRSkcBSERERCodJ7MLKIusVisHDx6kSpUqWCwWs8sRERGRS2AYBsePHycoKAgHhwuP8SgAFeHgwYOEhISYXYaIiIhcgX379nHNNddcsI0CUBGqVKkCFPwCvb29Ta5GRERELkV6ejohISG27/ELUQAqQuFhL29vbwUgERGRcuZSpq9oErSIiIhUOgpAIiIiUukoAImIiEilozlAVyE/P5/c3Fyzy5Bi5OzsjKOjo9lliIhICVMAugKGYZCUlMSxY8fMLkVKQNWqVQkICNA1oEREKjAFoCtQGH78/Pzw8PDQF2UFYRgGmZmZHD58GIDAwECTKxIRkZKiAHSZ8vPzbeGnRo0aZpcjxczd3R2Aw4cP4+fnp8NhIiIVlCZBX6bCOT8eHh4mVyIlpfDPVvO7REQqLgWgK6TDXhWX/mxFRCo+BSARERGpdBSAREREpNJRAJIijR49mrCwMLPLEBERKREKQJWAxWK54DJ69Ohztnn++eeJj4+3Pe/Tpw9du3YttZpDQ0Nt9bm7uxMaGkr37t358ccfAUhNTSUgIIDXXnvtnG27d+9O69atCQkJuWC/+/TpU2r9ERGRAoZhsHTHUvKseabWodPgK4FDhw7ZHi9YsIC4uDgSExNt67y8vGyPDcMgPz8fLy8vu/VmeOWVV+jXrx85OTn8888/fPTRR0RGRjJmzBhGjBjB7NmzefDBB+ncuTNNmjQB4PPPP+fbb79l48aNVK9enfz8fABWrVpFt27dSExMxNvbGzh9yruIiJSO3f/u5pklz/Dttm+Z3H4yz7Z51rRaFICKgWFAZmbpv6+HB1zKCUsBAQG2xz4+PlgsFtu6hIQEbr/9dhYvXsxLL73EH3/8wffff09CQgILFy5k06ZNjB49mvnz5wOnz5Bavnw57dq1448//mDQoEGsXr0aDw8PunXrxuTJk23hqU+fPhw7doxbbrmFN954g5ycHB566CGmTJmCs7PzBeuuUqWKrc5atWpx2223ERgYSFxcHA888ABdunTh4YcfJjo6mjVr1nDs2DGefvppxo8fT4MGDez2Vb16dQD8/PyoWrXqxX9pIiJSbLLzspm0ahJjfx5LVl4Wzg7OZOaa8MV5BgWgYpCZCWYMlpw4AZ6exbOvYcOGMWnSJOrWrUu1atVISEiwvfb888+zdetW0tPTmTt3LlAQKDIyMoiKiqJNmzb89ttvHD58mP/85z8MGDCAefPm2bZfvnw5gYGBLF++nB07dtCjRw/CwsLo16/fZdc5aNAgxowZw9dff82QIUN46623aNKkCWPGjGHr1q00btyYgQMHXu2vQ0REikn8rnieXvw0iUcKjjzcHno773R6h4a+DU2tSwFIgILDTXfddVeRr3l5eeHu7k52drbdaNL8+fPJysrigw8+wPNUEps2bRqdO3dmwoQJ+Pv7A1CtWjWmTZuGo6MjDRs2pFOnTsTHx19RAKpevTp+fn78888/AHh7ezN37lzat2+Pp6cnmzdv1nV8RETKgEPHD/Hc98/xyZZPAPD39Gdy1GR6Nu5ZJj6ny8Qk6OnTpxMaGoqbmxsRERGsXbv2krb79NNPsVgs50zONQyDuLg4AgMDcXd3JzIyku3bt5dA5QU8PApGY0p7Kc6LUYeHh1/2Nlu3bqVZs2a28ANw8803Y7Va7eYYNWrUyO6WEoGBgbb7bb322mu2+UZeXl7s3bv3ou9rGIbdP5477riD1q1b06tXL2rXrn3Z/RARkeKTZ83j7TVv03B6Qz7Z8gkOFgcGtBzA3wP+5uEmD5eJ8ANlYARowYIFxMbGMnPmTCIiIpgyZQpRUVEkJibi5+d33u3++ecfnn/+eW699dZzXps4cSJTp05l/vz51KlTh5EjRxIVFcVff/2Fm5tbsffBYim+Q1Fm8SzBDpw918disWC1WgHo378/3bt3t70WFBR0wX0dOXKElJQU6tSpY7feyckJJyfT/zqLiFRqa/av4cnvnmRj0kYAWga1ZEanGbQIamFyZecyfQRo8uTJ9OvXj5iYGG644QZmzpyJh4cHc+bMOe82+fn5PPLII7z88svUrVvX7jXDMJgyZQovvfQS9957L02bNuWDDz7g4MGDLFy4sIR7U3G5uLjYzqgqdP311/P777+TkZFhW7dy5UocHBzOmYR8PtWrV6devXq25WIh5q233sLBwaFUT8kXEZELO3ryKP2/7U+b99uwMWkjVd2qMqPTDFb3XV0mww+YHIBycnJYv349kZGRtnUODg5ERkayevXq8273yiuv4OfnR9++fc95bffu3SQlJdnt08fHh4iIiPPuMzs7m/T0dLtF7IWGhrJ582YSExNJTU0lNzeXRx55BDc3N6Kjo9myZQvLly9n4MCB9OrVyzb/52ocP36cpKQk9u3bx08//cTjjz/O2LFjefXVV6lXr14x9EpERK6GYRjM3zSfhtMaMmv9LAwMoptFkzggkf7h/XF0cLz4TkxiagBKTU0lPz//nC9Lf39/kpKSitzml19+4f333+fdd98t8vXC7S5nn+PGjcPHx8e2hISEXG5XKrx+/frRoEEDwsPDqVmzJitXrsTDw4OlS5dy9OhRWrZsyQMPPMCdd97JtGnTiuU9C+dx1atXj169epGWlkZ8fDxDhw4tlv2LiMiV23J4C23ntaXP131IyUzhhpo3sKLPCuZ1nYef5/mnsJQVFsMwDLPe/ODBgwQHB7Nq1SratGljWz9kyBBWrFjBmjVr7NofP36cpk2b8s4779CxY0fg9HVmCg9vrVq1iptvvpmDBw8SGBho27Z79+5YLBYWLFhwTh3Z2dlkZ2fbnqenpxMSEkJaWprtonmFsrKy2L17N3Xq1CmR+URiPv0Zi4ic34mcE7yc8DJv/vom+UY+Hs4ejG47msGtB+PseOHru5W09PR0fHx8ivz+Ppups0Z9fX1xdHQkOTnZbn1ycrLd6daFdu7cyT///EPnzp1t6won0zo5OZGYmGjbLjk52S4AJScnn/feVq6urri6ul5td0RERCoswzD46u+vGLRkEPvT9wNwX8P7mNJhCrV8aplc3eUz9RCYi4sLLVq0sLvnlNVqJT4+3m5EqFDDhg35448/2LRpk23p0qULt99+O5s2bSIkJIQ6deoQEBBgt8/09HTWrFlT5D5FRETkwnb9u4t7PrmHbp91Y3/6fupUrcO3Pb/lyx5flsvwA2XgNPjY2Fiio6MJDw+nVatWTJkyhYyMDGJiYgDo3bs3wcHBjBs3Djc3Nxo3bmy3feFtDc5cP3jwYMaOHUv9+vVtp8EHBQXpzCEREZHLkJ2XzcSVE3ntl9dst7AYevNQht86HA/nYrwYnQlMD0A9evQgJSWFuLg4kpKSCAsLY8mSJbZJzHv37sXB4fIGqoYMGUJGRgaPP/647T5US5Ys0XwOERGRS7Rs5zKeXvw0248WXEj4zjp3Mv3u6TTwvbTLnJR1pk6CLqsuNIlKE2QrPv0Zi0hldvD4QWKXxrLgz4KThgK8Angz6k16NOpRZq7ifD7lZhK0iIiIlA151jymr53OyOUjOZ5z3HYLi1dufwUfNx+zyyt2CkAiIiKV3Op9q3nyuyf5Pfl3ACKCI5jRaQbNA5ubXFnJUQASERGppI5kHmHYD8N4b+N7AFRzq8aEyAn0vbEvDhbT75ZVoip278Q0FotF914TESmjDMPgo80f0WBaA1v4iQmLIXFAIv1a9Kvw4QcUgCoFi8VywWX06NFmlwhAQkKCrSYHBwd8fHxo3rw5Q4YM4dChQwB8+OGHeHp6smPHDrttDx48SLVq1Zg2bdpF+5uQkGBC70REyobkE8l0+6wbvb7qxZGTR2js15ifY35mzr1zqOlZ0+zySo0OgVUCheEBYMGCBcTFxZGYmGhb5+XldVn7y83Nxdm55C53npiYiLe3N+np6WzYsIGJEyfy/vvvk5CQQK9evfjqq6/o06cPP/30k+0SCf369aNFixb069ePBx54wLavQYMGkZ6ezty5c23rqlevXmK1i4iUZf/96788+d2TpGam4uzgTFzbOIbePNT0W1iYQSNAlUBAQIBt8fHxwWKx2J77+fkxefJkrrnmGlxdXW3XYSr0zz//2O6h1rZtW9zc3Pi///s/AObMmUOjRo1wdXUlMDCQAQMG2L1vamoq9913Hx4eHtSvX59FixZdUr1+fn4EBARw3XXX8dBDD7Fy5Upq1qzJk08+CcCsWbPYtm0bkydPBmDevHmsXLmSuXPn4urqatdfd3f3c9a5uLgUx69VRKTcOJJ5hJ5f9OTBzx8kNTOVpv5N+a3fb7x020uVMvyARoCKhWEYZOZmlvr7ejh7XPU1Gd566y3eeOMNZs2aRfPmzZkzZw5dunThzz//pH79+rZ2w4YN44033qB58+a4ubkxY8YMYmNjGT9+PB07diQtLY2VK1fa7fvll19m4sSJvP7667z99ts88sgj7Nmz57JHYNzd3enfvz/PPvsshw8fxs/Pj9mzZ9OzZ0+aNWvGs88+y1tvvUVISMhV/S5ERCqib7d9S79v+pF0IglHiyPDbxnOyLYjcXGs3P8ZVAAqBpm5mXiNu7zDSMXhxPATeLp4XtU+Jk2axNChQ3nooYcAmDBhAsuXL2fKlClMnz7d1m7w4MHcf//9tudjx47lueeeY9CgQbZ1LVu2tNt3nz596NmzJwCvvfYaU6dOZe3atXTo0OGy62zYsCFQMCLl5+dH165d6d69Ox06dKBz585ER0df9j5FRCqytKw0Bi8dzLxN8wC43vd65nedT8vglhfesJLQIbBKLD09nYMHD3LzzTfbrb/55pvZunWr3brw8HDb48OHD3Pw4EHuvPPOC+6/adOmtseenp54e3tz+PBhABo1aoSXlxdeXl507NjxorUWXrD8zBGvkSNHYrVaeemlly66vYhIZbJs5zKazGjCvE3zsGDh+TbPs+GJDQo/Z9AIUDHwcPbgxPATprxvafH0PD3S5O7ufknbnD1R2mKxYLVaAVi8eDG5ubmXvL/CQBYaGmpb5+TkZPdTRKSyO5FzgiHLhjBj3QwArq12LfO6zuOWWreYXFnZo2+OYmCxWK76UJQZvL29CQoKYuXKlbRt29a2fuXKlbRq1eq821WpUoXQ0FDi4+O5/fbbr+i9a9eufcltT548yezZs7ntttuoWbPynKIpInI5ftrzEzFfx7Dr310ADGg5gPGR48vl91NpUACq5F544QVGjRrFtddeS1hYGHPnzmXTpk22M73OZ/To0fTv3x8/Pz86duzI8ePHWblyJQMHDrzqmg4fPkxWVhbHjx9n/fr1TJw4kdTUVL788sur3reISEVzMvckI34cwZRfp2BgUMunFnO6zOHOuheeplDZKQBVcs888wxpaWk899xzHD58mBtuuIFFixbZnQFWlOjoaLKysnjzzTd5/vnn8fX1tbv+ztVo0KABFosFLy8v6tatS/v27YmNjSUgIKBY9i8iUlGs2b+G6IXRJB4puLZb3+Z9mRw1GW/XC98JXcBiFM4uFZv09HR8fHxIS0vD29v+L1FWVha7d++mTp06uLm5mVShlCT9GYtIWZedl83ohNFMXDURq2El0CuQ97q8x9317za7NFNd6Pv7bBoBEhERKUc2HNpA9MJothzeAsAjTR5hasepVHfXVe4vhwKQiIhIOZCbn8trP7/G2J/HkmfNo6ZHTWbdM4v7rr/P7NLKJQUgERGRMu7Pw3/Se2FvNhzaAEC367sxo9OMSnXz0uKmACQiIlJG5VvzmbRqEnEJceTk51DNrRrT757OQ40fuupbIVV2CkAiIiJl0LYj2+izsA+r968GoFP9TszuPJugKkEmV1YxKACJiIiUIVbDyttr3mZ4/HBO5p3E29WbKVFT6BPWR6M+xUgBSEREpIzY/e9uHlv0GAn/JAAQWTeS97u8Ty2fWuYWVgEpAImIiJjMMAxmr5/N88ue50TOCTydPXn9rtfpH95foz4lRAFIRETERPvT99N3UV++3/k9ALfWupW5987l2urXmlxZxaYAJCIiYpIv/vqCft/049+sf3FzcuO1O15jUOtBOFgczC6twtNvuBLp06dgAp3FYsHZ2Zk6deowZMgQsrKybG0KX//111/tts3OzqZGjRpYLBYSEhJs61esWMEdd9xB9erV8fDwoH79+kRHR5OTkwNAQkICFouFRo0akZ+fb7fPqlWrMm/evBLrr4hIWZWRk0G/Rf144PMH+DfrX1oEtmDjExt5ts2zCj+lRL/lSqZDhw4cOnSIXbt28eabbzJr1ixGjRpl1yYkJIS5c+farfvqq6/w8vKyW/fXX3/RoUMHwsPD+emnn/jjjz94++23cXFxOSfs7Nq1iw8++KBY+1IYskREypP1B9dz4+wbeW/je1iwMOzmYazqu4qGvg3NLq1SUQCqZFxdXQkICCAkJISuXbsSGRnJsmXL7NpER0fz6aefcvLkSdu6OXPmEB0dbdfu+++/JyAggIkTJ9K4cWOuvfZaOnTowLvvvou7u7td24EDBzJq1Ciys7PPW9vevXu599578fLywtvbm+7du5OcnGx7ffTo0YSFhfHee+/Z3ajUYrEwa9Ys7rnnHjw8PLj++utZvXo1O3bsoF27dnh6enLTTTexc+fOK/69iYhcLathZeLKibR5vw3bjmwjuEow8b3jGRc5DhdHF7PLq3QUgIqDYUBeRukvhnFVZW/ZsoVVq1bh4mL/D69FixaEhobyxRdfAAXB5KeffqJXr1527QICAjh06BA//fTTRd9r8ODB5OXl8fbbbxf5utVq5d577+Xo0aOsWLGCZcuWsWvXLnr06GHXbseOHXzxxRd8+eWXbNq0ybZ+zJgx9O7dm02bNtGwYUMefvhhnnjiCYYPH866deswDIMBAwZcyq9FRKTYHUg/wF0f3sXQH4aSa82l2/Xd2PzkZm6vc7vZpVVamgRdHPIz4TOvi7crbt1PgJPnZW3y7bff4uXlRV5eHtnZ2Tg4ODBt2rRz2j322GPMmTOHRx99lHnz5nH33XdTs6b9PWcefPBBli5dStu2bQkICKB169bceeed9O7dG29vb7u2Hh4ejBo1ihdffJF+/frh4+Nj93p8fDx//PEHu3fvJiQkBIAPPviARo0a8dtvv9GyZUug4LDXBx98cE4tMTExdO/eHYChQ4fSpk0bRo4cSVRUFACDBg0iJibmsn5XIiLF4autX/Gfb/7D0ZNH8XD2YGqHqTzW/DGd3m6yMjECNH36dEJDQ3FzcyMiIoK1a9eet+2XX35JeHg4VatWxdPTk7CwMD788EO7NmdO9i1cOnToUNLdKBduv/12Nm3axJo1a4iOjiYmJoZu3bqd0+7RRx9l9erV7Nq1i3nz5vHYY4+d08bR0ZG5c+eyf/9+Jk6cSHBwMK+99hqNGjXi0KFD57Tv27cvNWrUYMKECee8tnXrVkJCQmzhB+CGG26gatWqbN261baudu3a54QfgKZNm9oe+/v7A9CkSRO7dVlZWaSnp5/vVyMiUqwycjJ4/JvHuf+z+zl68qhtonPfG/sq/JQBpo8ALViwgNjYWGbOnElERARTpkwhKiqKxMRE/Pz8zmlfvXp1RowYQcOGDXFxceHbb78lJiYGPz8/2//2oWCy75kTeV1dXUuuE44eBaMxpc3R47I38fT0pF69ekDBvJ5mzZrx/vvv07dvX7t2NWrU4J577qFv375kZWXRsWNHjh8/XuQ+g4OD6dWrF7169WLMmDFcd911zJw5k5dfftmunZOTE6+++ip9+vS54sNRnp5Fj3g5OzvbHhd+sBS1zmq1XtH7iohcjg2HNvDwFw+TeCQRCxZeuOkFxtwxRnN9yhDTR4AmT55Mv379iImJ4YYbbmDmzJl4eHgwZ86cItu3a9eO++67j+uvv55rr72WQYMG0bRpU3755Re7doWTfQuXatWqlVwnLJaCQ1GlvVzl/yAcHBx48cUXeemll+wmPBd67LHHSEhIoHfv3jg6Ol7SPqtVq0ZgYCAZGRlFvv7ggw/SqFGjc8LR9ddfz759+9i3b59t3V9//cWxY8e44YYbLqNXIiLmsRpWXl/5Oq3fa03ikUSCqgTxQ+8fmHDXBIWfMsbUAJSTk8P69euJjIy0rXNwcCAyMpLVq1dfdHvDMIiPjycxMZHbbrvN7rWEhAT8/Pxo0KABTz75JEeOHDnvfrKzs0lPT7dbKosHH3wQR0dHpk+ffs5rHTp0ICUlhVdeeaXIbWfNmsWTTz7J999/z86dO/nzzz8ZOnQof/75J507dz7ve44fP545c+bYhaTIyEiaNGnCI488woYNG1i7di29e/embdu2hIeHX31HRURK2IH0A7T/sD1DfhhCrjWX+xrex+b+m7mjzh1mlyZFMDUApaamkp+fb5uzUcjf35+kpKTzbpeWloaXlxcuLi506tSJt99+m7vuusv2eocOHfjggw+Ij49nwoQJrFixgo4dO55zbZpC48aNw8fHx7acOQ+lonNycmLAgAFMnDjxnFEbi8WCr6/vOWeJFWrVqhUnTpygf//+NGrUiLZt2/Lrr7+ycOFC2rZte973vOOOO7jjjjvIy8uze6+vv/6aatWqcdtttxEZGUndunVZsGBB8XRURKQELfx7IU1nNiV+dzwezh7Mvmc2X3T/ghoeNcwuTc7DYhhXeS71VTh48CDBwcGsWrWKNm3a2NYPGTKEFStWsGbNmiK3s1qt7Nq1ixMnThAfH8+YMWNYuHAh7dq1K7L9rl27uPbaa/nhhx+48847z3k9Ozvb7vo06enphISEkJaWds7ZTFlZWezevdvuOjRSsejPWEQuVUZOBrFLY5m9YTYANwbeyMf3f0wD3wYmV1Y5paen4+PjU+T399lMnQTt6+uLo6Oj3cXuAJKTkwkICDjvdg4ODraJvGFhYWzdupVx48adNwDVrVsXX19fduzYUWQAcnV1LdlJ0iIiUuFsPLSRnl/0JPFIIgAv3PQCY+8Yq7k+5YSph8BcXFxo0aIF8fHxtnVWq5X4+Hi7EaGLsVqtF7zC8P79+zly5AiBgYFXVa+IiIjVsPLGqjeIeC/i9ETnXj8w8a6JCj/liOmnwcfGxhIdHU14eDitWrViypQpZGRk2C5a17t3b4KDgxk3bhxQMF8nPDyca6+9luzsbBYvXsyHH37IjBkzADhx4gQvv/wy3bp1IyAggJ07dzJkyBDq1atnd5q8iIjI5Tp4/CB9FvZh2a6CWwh1bdiV9zq/p7k+5ZDpAahHjx6kpKQQFxdHUlISYWFhLFmyxDYxeu/evTg4nB6oysjI4KmnnmL//v24u7vTsGFDPvroI9stExwdHdm8eTPz58/n2LFjBAUF0b59e8aMGaPDXCIicsUWJS7isa8f48jJI7g7uTOlwxT63dhPFzUsp0ydBF1WXWgSlSbIVnz6MxaRM2XmZvLc0ueYuX4mAM0DmvNxt4919/YyqNxMghYRESnLNiVt4uEvHmZrasEteZ5v8zxj7xiLq5OOKJR3CkAiIiJnsRpWpvw6heHxw8nJzyHQK5D5Xedz17V3XXxjKRcUgERERM5w6Pgh+nzdh+93fg9AlwZdeL/L+/h6+JpcmRQnBSAREZFTvkn8hscWPUZqZiruTu5MjprMEy2e0ETnCsj0m6FK2WCxWFi4cKHZZYiImCIlI4W+X/ely6ddSM1MpZl/M9Y/vp7+4f0VfiooBaBKpE+fPnTt2rXI1w4dOkTHjh1Lt6BTQkNDsVgsWCwW3N3dCQ0NpXv37vz4449AwT3jAgICeO21187Ztnv37rRu3ZqQkBDbPopa+vTpU8q9EpHyIN+az4zfZtBgWgPmbJoDQGzrWNb8Zw3X17ze5OqkJOkQmABc8NYjxSUnJ+e8N1Z95ZVX6NevHzk5Ofzzzz989NFHREZGMmbMGEaMGMHs2bN58MEH6dy5M02aNAHg888/59tvv2Xjxo1Ur17ddrPbVatW0a1bNxITE22nQbq7u5d4/0SkfFmzfw1PLX6KDYc2ABAWEMb0u6dzU8hNJlcmpUEjQAKcewhs1apVhIWF4ebmRnh4OAsXLsRisbBp0yZbmy1bttCxY0e8vLzw9/enV69epKam2l5v164dAwYMYPDgwfj6+l7wStxVqlQhICCAWrVqcdtttzF79mxGjhxJXFwciYmJdOnShYcffpjo6Ghyc3NJSUnh6aefZvz48TRo0ICaNWsSEBBAQEAA1atXB8DPz8+2zsfHp9h/ZyJSPqVmptJvUT9av9+aDYc24OPqw9sd3+a3fr8p/FQiCkDFwTAgI6P0lxK6hmV6erptpGXDhg2MGTOGoUOH2rU5duwYd9xxB82bN2fdunUsWbKE5ORkunfvbtdu/vz5uLi4sHLlSmbOnHlZdQwaNAjDMPj6668BeOuttzhy5AhjxozhqaeeonHjxgwcOPDqOisilUa+NZ9Z62Zx3dvX8d7G9wDoE9aHxAGJDGg1ACcHHRSpTPSnXRwyM8HLq/Tf98QJ8PQs9t1+/PHHWCwW3n33Xdzc3Ljhhhs4cOAA/fr1s7WZNm0azZs3t5uXM2fOHEJCQti2bRvXXXcdAPXr12fixIlXVEf16tXx8/Pjn3/+AcDb25u5c+fSvn17PD092bx5syYnisglWXtgLU8vfpp1B9cB0My/GdPvns7NtW42uTIxi0aA5ByJiYk0bdrU7jYQrVq1smvz+++/s3z5cry8vGxLw4YFl4XfuXOnrV2LFi1sj1977TW79nv37r1oLYZh2IWcO+64g9atW9OrVy9q1659xX0UkcohNTOVx795nNbvtWbdwXV4u3oztcNU1j2+TuGnktMIUHHw8CgYjTHjfU1y4sQJOnfuzIQJE855LTAw0PbY84wRqv79+9sdIgsKCrrgexw5coSUlBTq1Kljt97JyQknJ/3VFZHzy7fm8/7G9xkeP5yjJ48C0LtZbyZGTsTfy9/k6qQs0LdIcbBYSuRQlFkaNGjARx99RHZ2Nq6uBfe7+e233+za3HjjjXzxxReEhoZechipXr26bYLypXjrrbdwcHA476n7IiJF+e3Abzy9+Gl+O1jwudXUvynT757OLbVuMbkyKUt0CKySSUtLY9OmTXbLvn377No8/PDDWK1WHn/8cbZu3crSpUuZNGkSgO1w1NNPP83Ro0fp2bMnv/32Gzt37mTp0qXExMTYTke/HMePHycpKYl9+/bx008/8fjjjzN27FheffVV6tWrd/UdF5EK70jmEZ745gki3ovgt4O/4e3qzVsd3mL94+sVfuQcGgGqZBISEmjevLndur59+9o99/b25ptvvuHJJ58kLCyMJk2aEBcXx8MPP2ybFxQUFMTKlSsZOnQo7du3Jzs7m9q1a9OhQwccHC4/V8fFxREXF4eLiwsBAQG0bt2a+Ph4br/99ivvrIhUClbDynsb3rM73NWraS8m3jWRAK+Sv8aZlE8Wwyihc6nLsfT0dHx8fEhLS7NdSK9QVlYWu3fvpk6dOnaThCu6//u//yMmJoa0tLQKf1HByvpnLFIenX24q4lfE6bdPY3bat9mcmVihgt9f59NI0BSpA8++IC6desSHBzM77//ztChQ+nevXuFDz8iUj4cyTzCiB9HMHv9bAwMvF29eaXdKzzd6mldz0cuif6WSJGSkpKIi4sjKSmJwMBAHnzwQV599VWzyxKRSs5qWJmzcQ7DfhjGkZNHAB3ukiujACRFGjJkCEOGDDG7DBERm/UH1/PU4qdYe2AtAI39GjP97uk63CVXRAFIRETKtKMnjzIifgSz1s/CwKCKSxVeuf0Vnm75NM6OzmaXJ+WUAtAV0tzxikt/tiJlg9WwMnfjXIb+MNR2uOuRJo/w+l2vE1gl8CJbi1yYAtBlcnYu+N9GZmamJgRXUJmZmcDpP2sRKX0bDm3gqe+eYs2BNQA0qtmI6XdPp21oW5Mrk4pCAegyOTo6UrVqVQ4fPgyAh4eHbshZQRiGQWZmJocPH6Zq1ao4OjqaXZJIpZOZm8nwH4bz9tq3bYe7Xm73MgNaDdDhLilWCkBXICCg4EyDwhAkFUvVqlVtf8YiUnp+3f8r0Quj2XZkGwAPN3mY1+96naAqF75voMiVUAC6AhaLhcDAQPz8/MjNzTW7HClGzs7OGvkRKWU5+Tm8nPAy41eOx2pYCa4SzPtd3ieqXpTZpUkFpgB0FRwdHfVlKSJyFTYnb6b3V735Pfl3AB5t+ihTO0ylmns1kyuTik4BSERESl2+NZ/XV71O3PI4cq25+Hr4MrPTTLrd0M3s0qSSUAASEZFStf3IdqIXRrN6/2oA7m1wL7PumYW/l7/JlUllogAkIiKlwmpYmfHbDIb8MITM3Ey8Xb15q8NbRDeL1tm0UuoUgEREpMTtS9vHY4se44ddPwBwR507mHvvXGr51DK5MqmsHMwuAGD69OmEhobi5uZGREQEa9euPW/bL7/8kvDwcKpWrYqnpydhYWF8+OGHdm0MwyAuLo7AwEDc3d2JjIxk+/btJd0NERE5i2EYfPD7BzSZ0YQfdv2Au5M7UztMZVmvZQo/YirTA9CCBQuIjY1l1KhRbNiwgWbNmhEVFXXea+xUr16dESNGsHr1ajZv3kxMTAwxMTEsXbrU1mbixIlMnTqVmTNnsmbNGjw9PYmKiiIrK6u0uiUiUukdzjjM/Z/dT/TCaNKy04gIjmBT/00MjBiIg8X0rx+p5CyGyTc+ioiIoGXLlkybNg0Aq9VKSEgIAwcOZNiwYZe0jxtvvJFOnToxZswYDMMgKCiI5557jueffx6AtLQ0/P39mTdvHg899NBF95eeno6Pjw9paWl4e3tfeedERCqpr7Z+xRPfPkFKZgrODs6MbjeaITcPwclBMy+k5FzO97epETwnJ4f169cTGRlpW+fg4EBkZCSrV6++6PaGYRAfH09iYiK33XYbALt37yYpKclunz4+PkRERJx3n9nZ2aSnp9stIiJy+Y5lHaP3V725/7P7SclMoYlfE37r9xsv3vqiwo+UKab+bUxNTSU/Px9/f/tTH/39/fn777/Pu11aWhrBwcFkZ2fj6OjIO++8w1133QVAUlKSbR9n77PwtbONGzeOl19++Wq6IiJS6S3buYzHFj3G/vT9OFgcGHLTEEa3G42rk6vZpYmco1zG8SpVqrBp0yZOnDhBfHw8sbGx1K1bl3bt2l3R/oYPH05sbKzteXp6OiEhIcVUrYhIxZaRk8GQZUN4Z907ANSrXo/5XedzU8hNJlcmcn6mBiBfX18cHR1JTk62W5+cnHzBm1E6ODhQr149AMLCwti6dSvjxo2jXbt2tu2Sk5MJDAy022dYWFiR+3N1dcXVVf9DERG5XKv2rSJ6YTQ7ju4A4OmWTzMhcgKeLp4mVyZyYabOAXJxcaFFixbEx8fb1lmtVuLj42nTps0l78dqtZKdnQ1AnTp1CAgIsNtneno6a9asuax9iojI+WXnZTP8h+HcOvdWdhzdQXCVYL5/9Hum3T1N4UfKBdMPgcXGxhIdHU14eDitWrViypQpZGRkEBMTA0Dv3r0JDg5m3LhxQMF8nfDwcK699lqys7NZvHgxH374ITNmzAAK7tQ+ePBgxo4dS/369alTpw4jR44kKCiIrl27mtVNEZEK4/ek3+n1VS/+OPwHAL2a9mJqx6lUdatqbmEil8H0ANSjRw9SUlKIi4sjKSmJsLAwlixZYpvEvHfvXhwcTg9UZWRk8NRTT7F//37c3d1p2LAhH330ET169LC1GTJkCBkZGTz++OMcO3aMW265hSVLluDm5lbq/RMRqSjyrHlMXDmR0QmjybXmUtOjJrPumcV9199ndmkil8306wCVRboOkIiIvW1HttH7q96sObAGgK4NuzLrnln4efqZXJnIaZfz/W36CJCIiJRdVsPK9LXTGfrDUE7mncTb1Zu3O75Nr6a9dANTKdcUgEREpEh70/YS83UMP+7+EYDIupHM6TKHEB9dJkTKPwUgERGxYxgG7214j+e+f47jOcdxd3Ln9bte58mWT+oeXlJhKACJiIjNnmN7+M83/+GHXT8A0OaaNszrOo/ralxncmUixUsBSEREsBpWZq+fzQvLXuBEzgncnNx49Y5XGRQxCEcHR7PLEyl2CkAiIpXc7n9303dRX5b/sxyAW2rdwpwuc6hfo77JlYmUHAUgEZFKympYmfHbDIb+MJSM3AzcndwZHzmeAa0GaK6PVHgKQCIildDOozvpu6gvK/asAOC22rfxfpf3qVe9nsmViZQOBSARkUrEaliZtnYaw+OHk5mbiYezBxMiJ/BUy6c06iOVigKQiEglsf3Idvou6svPe38GoF1oO97v8j51q9U1uTKR0qcAJCJSweVb85m6ZiojfhzBybyTeLl4MTFyIk+EP6FRH6m0FIBERCqwxNREHlv0GKv2rQLgzjp38l6X9witGmpuYSImUwASEamA8q35TPl1Ci8tf4msvCyquFRhUvtJ9Luxn+7hJYICkIhIhfN36t/EfB3Dr/t/BaD9te15t/O71PKpZXJlImWHApCISAWRZ81j8urJxC2PIzs/G29Xbya3n8xjzR/TqI/IWRSAREQqgL9S/iLm6xjWHlgLQMd6HZl1zyzduV3kPBSARETKsTxrHq+vfJ3RK0aTk5+Dj6sPUzpMIbpZtEZ9RC5AAUhEpJzacngLMV/HsO7gOgA61e/ErHtmEewdbHJlImWfApCISDmTm5/LhJUTeGXFK+Rac6nqVpWpHabyaNNHNeojcokUgEREypHNyZvps7APG5M2AtClQRdmdppJYJVAkysTKV8UgEREyoGc/BzG/TyOsT+PJc+aR3X36rzd8W16Nu6pUR+RK6AAJCJSxm1K2kSfhX34Pfl3AO5reB/vdHqHAK8AkysTKb8UgEREyijDMJi6ZipDfhhCTn4ONdxrMP3u6XRv1F2jPiJXSQFIRKQMSs1M5bGvH+Obbd8AcG+De5ndeTZ+nn4mVyZSMSgAiYiUMSv+WcEjXz7CgeMHcHV05Y32b/BUy6c06iNSjBSARETKiDxrHmN/GsuYn8ZgNaw0qNGABQ8soFlAM7NLE6lwFIBERMqA/en7eeTLR/hpz08APBb2GFM7TsXTxdPkykQqJgUgERGTLUpcRMzXMRw9eZQqLlWYec9MHm7ysNlliVRoCkAiIibJzstmyLIhTF07FYAWgS349IFPqVe9nsmViVR8CkAiIibYdmQbD/33IdsVnZ9r8xyv3fkaLo4uJlcmUjkoAImIlLIPfv+Ap757iozcDHw9fJnfdT5317/b7LJEKhUHswsAmD59OqGhobi5uREREcHatWvP2/bdd9/l1ltvpVq1alSrVo3IyMhz2vfp0weLxWK3dOjQoaS7ISJyQcezj9P7q95EL4wmIzeD20Nv5/f+vyv8iJjA9AC0YMECYmNjGTVqFBs2bKBZs2ZERUVx+PDhItsnJCTQs2dPli9fzurVqwkJCaF9+/YcOHDArl2HDh04dOiQbfnkk09KozsiIkXacGgDLWa34MPNH+JgcWDM7WNY1msZQVWCzC5NpFKyGIZhmFlAREQELVu2ZNq0aQBYrVZCQkIYOHAgw4YNu+j2+fn5VKtWjWnTptG7d2+gYATo2LFjLFy48JJqyM7OJjs72/Y8PT2dkJAQ0tLS8Pb2vvxOiYiccvbtLEK8Q/i428fcUusWs0sTqXDS09Px8fG5pO9vU0eAcnJyWL9+PZGRkbZ1Dg4OREZGsnr16kvaR2ZmJrm5uVSvXt1ufUJCAn5+fjRo0IAnn3ySI0eOnHcf48aNw8fHx7aEhIRcWYdERM6QmpnKvZ/ey+Clg8nJz6Frw65s6r9J4UekDDA1AKWmppKfn4+/v7/den9/f5KSki5pH0OHDiUoKMguRHXo0IEPPviA+Ph4JkyYwIoVK+jYsSP5+flF7mP48OGkpaXZln379l15p0REKLidRdjMML7Z9g2ujq5M6ziNL7t/SXX36hffWERKXLk+C2z8+PF8+umnJCQk4ObmZlv/0EMP2R43adKEpk2bcu2115KQkMCdd955zn5cXV1xdXUtlZpFpGLLt+Yz5qcxup2FSBln6giQr68vjo6OJCcn261PTk4mICDggttOmjSJ8ePH8/3339O0adMLtq1bty6+vr7s2LHjqmsWETmf/en7ueODO3h5xctYDSsxYTGsf3y9wo9IGWRqAHJxcaFFixbEx8fb1lmtVuLj42nTps15t5s4cSJjxoxhyZIlhIeHX/R99u/fz5EjRwgMDCyWukVEzrYocRHNZjbjpz0/4eXixUf3fcSce+foXl4iZZTph8BiY2OJjo4mPDycVq1aMWXKFDIyMoiJiQGgd+/eBAcHM27cOAAmTJhAXFwcH3/8MaGhoba5Ql5eXnh5eXHixAlefvllunXrRkBAADt37mTIkCHUq1ePqKgo0/opIhWTbmchUj6ZHoB69OhBSkoKcXFxJCUlERYWxpIlS2wTo/fu3YuDw+mBqhkzZpCTk8MDDzxgt59Ro0YxevRoHB0d2bx5M/Pnz+fYsWMEBQXRvn17xowZo3k+IlKszr6dRWzrWMZFjtPtLETKAdOvA1QWXc51BESkcjr7dhbz7p1Hp+s6mV2WSKV2Od/fpo8AiYiUJ8ezj/P04qf5cPOHALQLbcdH931EsHewyZWJyOVQABIRuUQbDm3gof8+xPaj23GwOPByu5cZfstwHB0czS5NRC6TApCIyEVk5GQwOmE0b/76JvlGPtd4X8PH93/MrbVvNbs0EblCCkAiIhewKHERA/83kL1pewF44IYHmHXPLF3RWaScUwASESnCvrR9PLPkGRb+vRCA2j61mX73dE10FqkgFIBERM6QZ81j6pqpxC2PIyM3AycHJ55r8xwjbxupixqKVCAKQCIip6w9sJYnvn2CTUmbALgp5CZmdppJE/8m5hYmIsVOAUhEKr20rDRejH+RGetmYGBQza0aE++ayGPNH8PBYuodg0SkhCgAiUilZRgGC/5cwLNLnyXpRMFtdXo17cWk9pPw8/QzuToRKUkKQCJSKe08upOnFj/F9zu/B+C6Gtcxs9NMbq9zu8mViUhpUAASkUolOy+b11e9zqs/v0pWXhaujq68eOuLDL15KK5Oul+gSGWhACQilcaKf1bQ/7v+/J36NwCRdSN55+53qF+jvsmViUhpUwASkQovNTOV579/nvm/zwfAz9OPN6PepGfjnlgsFpOrExEzKACJSIVlGAZzN83lhWUvcPTkUQCeaPEE4+4cRzX3aiZXJyJmUgASkQrpr5S/6P9tf37e+zMATf2bMrPTTNqEtDG5MhEpCxSARKRCyczN5NWfXuX1Va+Ta83Fw9mDl9u9zKCIQTg7OptdnoiUEQpAIlJhLNmxhKe+e4rdx3YD0KVBF97u+Da1fGqZXJmIlDUKQCJS7h08fpBnlz7LZ39+BsA13tfwdse36dqwq7mFiUiZpQAkIuVWvjWfGetmMOLHEaRnp+NocWRQxCBevv1lvFy8zC5PRMowBSARKZc2HNrAE98+wbqD6wBoFdyKWffMIiwgzNzCRKRcUAASkXIlMzeTl358ibfWvIXVsOLj6sO4O8fxeIvHcXRwNLs8ESknFIBEpNz4ec/PxHwdw85/dwLwUOOHeDPqTQK8AkyuTETKGwUgESnzMnMzGRE/grfWvIWBwTXe1/Bu53fpUK+D2aWJSDmlACQiZdove3/hsa8fY/vR7QD0bd6XN9q/gY+bj8mViUh55nAlG82fP5/vvvvO9nzIkCFUrVqVm266iT179hRbcSJSeWXmZvLskme5be5tbD+6neAqwSx+eDHvdXlP4UdErtoVBaDXXnsNd3d3AFavXs306dOZOHEivr6+PPvss8VaoIhUPiv3riRsZhhT1kzBwCAmLIYtT22hY/2OZpcmIhXEFR0C27dvH/Xq1QNg4cKFdOvWjccff5ybb76Zdu3aFWd9IlKJnMw9yUs/vsSbv76JgUFwlWBmd57N3fXvNrs0EalgrmgEyMvLiyNHjgDw/fffc9dddwHg5ubGyZMni686Eak0Vu1bRdisMCb/Otlu1EfhR0RKwhWNAN1111385z//oXnz5mzbto277y74gPrzzz+pXbt2sRYoIhXb2aM+QVWCeLfzuwo+IlKirmgEaPr06bRp04aUlBS++OILatSoAcD69et5+OGHi7VAEam4Vu9bbTfqE90smi1PatRHREreFQWgqlWrMmnSJEaMGEFeXh6LFi1i0aJFtGjRgiZNmlz2/qZPn05oaChubm5ERESwdu3a87Z99913ufXWW6lWrRrVqlUjMjLynPaGYRAXF0dgYCDu7u5ERkayffv2y65LRErGydyTvPD9C9wy9xa2HdlGoFcg3/b8lnld51HNvZrZ5YlIJXBFh8CWLFlC7969OXLkCIZh2L1msVjIz8+/5H0tWLCA2NhYZs6cSUREBFOmTCEqKorExET8/PzOaZ+QkEDPnj256aabcHNzY8KECbRv354///yT4OBgACZOnMjUqVOZP38+derUYeTIkURFRfHXX3/h5uZ2JV0WkWLy6/5f6bOwD4lHEgHo3aw3U6KmKPiISKmyGGcnmEtQv3592rdvT1xcHP7+/ldVQEREBC1btmTatGkAWK1WQkJCGDhwIMOGDbvo9vn5+VSrVo1p06bRu3dvDMMgKCiI5557jueffx6AtLQ0/P39mTdvHg899NA5+8jOziY7O9v2PD09nZCQENLS0vD29r6q/olIgay8LOKWx/HG6jewGlYCvQKZdc8sOjfobHZpIlJBpKen4+Pjc0nf31d0CCw5OZnY2NirDj85OTmsX7+eyMjI0wU5OBAZGcnq1asvaR+ZmZnk5uZSvXp1AHbv3k1SUpLdPn18fIiIiDjvPseNG4ePj49tCQkJuYpeicjZft3/K81nNef1Va9jNaz0atqLP5/6U+FHRExzRQHogQceICEh4arfPDU1lfz8/HOClL+/P0lJSZe0j6FDhxIUFGQLPIXbXc4+hw8fTlpamm3Zt2/f5XZFRIqQlZfF0GVDuXnOzfyd+jcBXgEsemgRH9z3gQ55iYiprmgO0LRp03jwwQf5+eefadKkCc7OznavP/PMM8VS3MWMHz+eTz/9lISEhKua2+Pq6oqrq2sxViYiaw+spc/CPmxN3QrAo00f5a0Ob1HdvbrJlYmIXGEA+uSTT/j+++9xc3MjISEBi8Vie81isVxyAPL19cXR0ZHk5GS79cnJyQQEBFxw20mTJjF+/Hh++OEHmjZtaltfuF1ycjKBgYF2+wwLC7ukukTkymXlZTE6YbTtcFeAVwCz7plFlwZdzC5NRMTmig6BjRgxgpdffpm0tDT++ecfdu/ebVt27dp1yftxcXGhRYsWxMfH29ZZrVbi4+Np06bNebebOHEiY8aMYcmSJYSHh9u9VqdOHQICAuz2mZ6ezpo1ay64TxG5emsPrKXF7BZMWDkBq2HlkSaP8OdTfyr8iEiZc0UjQDk5OfTo0QMHhyvKT3ZiY2OJjo4mPDycVq1aMWXKFDIyMoiJiQGgd+/eBAcHM27cOAAmTJhAXFwcH3/8MaGhobZ5PV5eXnh5eWGxWBg8eDBjx46lfv36ttPgg4KC6Nq161XXKyLnys7LZnTCaCaumojVsOLv6c+se2Zxb8N7zS5NRKRIVxSAoqOjWbBgAS+++OJVF9CjRw9SUlKIi4sjKSmJsLAwlixZYpvEvHfvXrugNWPGDHJycnjggQfs9jNq1ChGjx4NwJAhQ8jIyODxxx/n2LFj3HLLLSxZskTXABIpAb8d+I0+X/fhr5S/AHi4ycNM7TCVGh41TK5MROT8rug6QM888wwffPABzZo1o2nTpudMgp48eXKxFWiGy7mOgEhllZ2XzSsrXmHCygnkG/n4efox655ZdG3Y1ezSRKSSupzv7ysaAfrjjz9o3rw5AFu2bLF77cwJ0SJSMW04tIHohdFsOVzw779n45683fFtjfqISLlxRQFo+fLlxV2HiJQDOfk5vPrTq7z686vkG/nU9KjJzHtmcv/195tdmojIZbmiACQilc/vSb/T5+s+bEraBMCDNzzI9LunU9OzprmFiYhcAQUgEbmg3PxcJqycwCsrXiHXmksN9xq80+kdujfqbnZpIiJXTAFIRM7rz8N/Er0wmvWH1gPQtWFXZnaaib/X1d0HUETEbApAInKOfGs+k1ZNIi4hjpz8HKq6VWVax2k83ORhneggIhWCApCI2ElMTaTP1334df+vAHSq34nZnWcTVCXI5MpERIqPApCIAAWjPlPXTOXFH18kKy8Lb1dv3urwFtHNojXqIyIVjgKQiLDj6A5ivo7hl72/AND+2va81/k9QnxCTK5MRKRkKACJVGJWw8r0tdMZ+sNQTuadxMvFi8ntJ/OfG/+jUR8RqdAUgEQqqd3/7uaxRY+R8E8CAHfUuYM5XeZQu2ptcwsTESkFCkAilYxhGMxeP5vnlz3PiZwTeDh78Ppdr9M/vD8OFoeL70BEpAJQABKpRPam7aXvor78sOsHAG6tdStz753LtdWvNbkyEZHSpQAkUgkYhsHcTXN5dumzpGen4+7kzrg7xzEwYqBGfUSkUlIAEqngDqQf4PFvH2fx9sUAtLmmDfO6zuO6GteZXJmIiHkUgEQqKMMw+GjzRzyz5BmOZR3D1dGVsXeM5dnWz+Lo4Gh2eSIiplIAEqmAkk4k8cS3T7AocREALYNaMq/rPG6oeYPJlYmIlA0KQCIViGEYLPhzAU8vfpqjJ4/i7ODMy+1e5oWbX8DJQf/cRUQK6RNRpIJIyUjhye+e5IutXwDQPKA587vOp4l/E5MrExEpexSARCqAL/76gie/e5KUzBScHJwYedtIht8yHGdHZ7NLExEpkxSARMqxwxmHGbB4AJ//9TkATfyaML/rfJoHNje5MhGRsk0BSKQcKpzrM/B/A0nNTMXR4siwW4YR1zYOF0cXs8sTESnzFIBEyplDxw/x1OKnWPj3QgCa+Tdj7r1zNeojInIZFIBEygnDMPhw84cMXjKYf7P+xdnBmZdue4lhtwzTqI+IyGVSABIpB/an7+eJb5+wXc25RWAL5t47V2d4iYhcIQUgkTLMMAzmbJxD7PexpGen4+Lowui2o3VdHxGRq6RPUJEyas+xPfT7ph/Ldi0DoPU1rZnTZQ7X17ze5MpERMo/BSCRMsZqWJm1bhZDfhjCiZwTuDm5Mfb2sQxuPVj38BIRKSYKQCJlyK5/d9F3UV8S/kkA4JZat/B+l/d153YRkWKmACRSBlgNK2+veZsXf3yRzNxMPJw9GH/neJ5u9TQOFgezyxMRqXBM/2SdPn06oaGhuLm5ERERwdq1a8/b9s8//6Rbt26EhoZisViYMmXKOW1Gjx6NxWKxWxo2bFiCPRC5OtuObOO2ubcxeOlgMnMzuT30dv548g8GRgxU+BERKSGmfrouWLCA2NhYRo0axYYNG2jWrBlRUVEcPny4yPaZmZnUrVuX8ePHExAQcN79NmrUiEOHDtmWX375paS6IHLF8q35TFo1iWYzm7Fy30q8XLyY0WkGP/T+gbrV6ppdnohIhWbqIbDJkyfTr18/YmJiAJg5cybfffcdc+bMYdiwYee0b9myJS1btgQo8vVCTk5OFwxIImb7K+UvYr6OYe2BghHP9te2593O71LLp5bJlYmIVA6mjQDl5OSwfv16IiMjTxfj4EBkZCSrV6++qn1v376doKAg6tatyyOPPMLevXsv2D47O5v09HS7RaQk5Obn8trPr9F8VnPWHliLj6sP73d5nyWPLFH4EREpRaYFoNTUVPLz8/H397db7+/vT1JS0hXvNyIignnz5rFkyRJmzJjB7t27ufXWWzl+/Ph5txk3bhw+Pj62JSQk5IrfX+R8NidvpvX7rRnx4why8nPoVL8Tfz71J481fwyLxWJ2eSIilUqFOwusY8eOtsdNmzYlIiKC2rVr89lnn9G3b98itxk+fDixsbG25+np6QpBUmxy8nN47efXePXnV8mz5lHNrRpTO07lkSaPKPiIiJjEtADk6+uLo6MjycnJduuTk5OLdf5O1apVue6669ixY8d527i6uuLq6lps7ylSaP3B9Ty26DE2J28G4L6G9/FOp3cI8NIcNRERM5l2CMzFxYUWLVoQHx9vW2e1WomPj6dNmzbF9j4nTpxg586dBAYGFts+RS4mOy+bF+NfJOK9CDYnb8bXw5dPu33KF92/UPgRESkDTD0EFhsbS3R0NOHh4bRq1YopU6aQkZFhOyusd+/eBAcHM27cOKBg4vRff/1le3zgwAE2bdqEl5cX9erVA+D555+nc+fO1K5dm4MHDzJq1CgcHR3p2bOnOZ2USmfN/jXEfB3D1tStAHRv1J1pHadR07OmyZWJiEghUwNQjx49SElJIS4ujqSkJMLCwliyZIltYvTevXtxcDg9SHXw4EGaN29uez5p0iQmTZpE27ZtSUhIAGD//v307NmTI0eOULNmTW655RZ+/fVXatbUl4+UrKy8LEb+OJLJv07Galjx9/TnnU7vcP/195tdmoiInMViGIZhdhFlTXp6Oj4+PqSlpeHt7W12OVIOrDu4jt5f9baN+jza9FGmRE2hhkcNkysTEak8Luf7u8KdBSZSmnLzcxn701he/flV8o18ArwCmH3PbDo36Gx2aSIicgEKQCJXaMvhLfT+qjcbkzYC0KNRD6bfPV2jPiIi5YACkMhlyrfm88bqNxi5fCQ5+TlUd6/OO3e/Q4/GPcwuTURELpECkMhl2HF0B30W9mHlvpUAdKrfiXc7v0tgFV1mQUSkPFEAErkEhmEwY90MXlj2Apm5mVRxqcKUDlOICYvR1ZxFRMohBSCRi9iXto++i/qybNcyAG4PvZ05984htGqouYWJiMgVUwASOQ/DMPhw84c8879nSMtOw83JjQmRExjQagAOFtMuoi4iIsVAAUikCMknkun/XX8W/r0QgIjgCOZ3nU8D3wbmFiYiIsVCAUjkLF/89QX9v+tPamYqzg7OvNzuZV64+QWcHPTPRUSkotAnusgp/578lwH/G8DHf3wMQFP/pnzQ9QOaBTQzuTIRESluCkAiwJIdS+i7qC8Hjx/EweLAsJuHMardKFwcXcwuTURESoACkFRqx7OP8/z3zzN7w2wArqtxHR90/YCIayJMrkxEREqSApBUWj/t+Yk+C/uw+9huAJ5p9QzjIsfh4exhcmUiIlLSFICk0jmZe5IRP45gyq9TMDCo7VObuffO5fY6t5tdmoiIlBIFIKlUfjvwG9ELo9mauhWAvs37MjlqMt6u3iZXJiIipUkBSCqFnPwcxv40ltd+fo18I58ArwDe7fwu91x3j9mliYiICRSApMLbcngLvb/qzcakjQD0aNSD6XdPp4ZHDZMrExERsygASYWVb83njdVvMHL5SHLyc6juXp137n6HHo17mF2aiIiYTAFIKqQdR3cQvTCaVftWAXDPdffwbud3CfAKMLkyEREpCxSApML55I9P+M83/yEzN5MqLlWY0mEKMWExWCwWs0sTEZEyQgFIKow8ax5Dlw1l8q+TAWgX2o55986jdtXaJlcmIiJljQKQVAgpGSn0+G8Plv+zHIDhtwxnzO1jcHRwNLkyEREpixSApNxbf3A99y24j33p+/B09mR+1/l0u6Gb2WWJiEgZpgAk5dq8TfPo/21/svOzqV+9Pl/1+IpGfo3MLktERMo4BSApl3Lyc4hdGsv036YDBWd5fXjfh1R1q2puYSIiUi4oAEm5k3QiiQc/f5Bf9v4CwKi2o4hrG4eDxcHkykREpLxQAJJy5df9v9Lts24cPH4Qb1dvPrzvQ7o06GJ2WSIiUs4oAEm5MXv9bAYsHkCuNZfrfa/nqx5f0cC3gdlliYhIOaQAJGVedl42A/83kHc3vAvA/dffz7x751HFtYrJlYmISHmlACRl2oH0A3T7rBtrDqzBgoVX73iVYbcM01WdRUTkqpg+a3T69OmEhobi5uZGREQEa9euPW/bP//8k27duhEaGorFYmHKlClXvU8pu37e8zMtZrdgzYE1VHOrxuJHFjP81uEKPyIictVMDUALFiwgNjaWUaNGsWHDBpo1a0ZUVBSHDx8usn1mZiZ169Zl/PjxBAQUfVPLy92nlD2GYTBt7TTu+OAOkjOSaerflHWPr6NDvQ5mlyYiIhWExTAMw6w3j4iIoGXLlkybNg0Aq9VKSEgIAwcOZNiwYRfcNjQ0lMGDBzN48OBi22eh9PR0fHx8SEtLw9vb+/I7JlfsZO5J+n/Xnw9+/wCAhxo/xHud38PTxdPkykREpKy7nO9v00aAcnJyWL9+PZGRkaeLcXAgMjKS1atXl+o+s7OzSU9Pt1uk9O05todb5t7CB79/gIPFgTfav8HH93+s8CMiIsXOtACUmppKfn4+/v7+duv9/f1JSkoq1X2OGzcOHx8f2xISEnJF7y9X7sfdP9Jidgs2HNqAr4cvy3otI7ZNrOb7iIhIiTB9EnRZMHz4cNLS0mzLvn37zC6p0jAMgzdWvcFdH97FkZNHaBHYgvWPr+eOOneYXZqIiFRgpp0G7+vri6OjI8nJyXbrk5OTzzvBuaT26erqiqur6xW9p1y5jJwM/vPNf/h0y6cARDeLZkanGbg7u5tcmYiIVHSmjQC5uLjQokUL4uPjbeusVivx8fG0adOmzOxTSsbOoztp834bPt3yKU4OTkzrOI25985V+BERkVJh6oUQY2NjiY6OJjw8nFatWjFlyhQyMjKIiYkBoHfv3gQHBzNu3DigYJLzX3/9ZXt84MABNm3ahJeXF/Xq1bukfYr5luxYQs8venIs6xj+nv58/uDn3Fr7VrPLEhGRSsTUANSjRw9SUlKIi4sjKSmJsLAwlixZYpvEvHfvXhwcTg9SHTx4kObNm9ueT5o0iUmTJtG2bVsSEhIuaZ9iHsMwGPfLOF768SUMDCKCI/ii+xcEewebXZqIiFQypl4HqKzSdYCK3/Hs4/T5ug9fbv0SgMdvfJypHafi6qS5VyIiUjwu5/tb9wKTEpeYmsh9C+5ja+pWXBxdmNZxGv1a9DO7LBERqcQUgKREfZP4DY9+9Sjp2ekEVwnmv93/S+trWptdloiIVHK6DpCUiDxrHiPiR9Dl0y6kZ6dza61bWf/4eoUfEREpEzQCJMXu0PFD9PyiJyv2rABgYKuBvNH+DZwdnU2uTEREpIACkBSr5buX0/OLniRnJOPl4sV7nd+jR+MeZpclIiJiRwFIioXVsPLaz68xKmEUVsNKE78m/Lf7f7muxnVmlyYiInIOBSC5aikZKfT6qhdLdy4F4LGwx3j77rfxcPYwuTIRESkJhgHZ2ZCefno5fvz8z4t6rX9/eOYZ8/qgACRXZdW+VXT/vDsHjh/A3cmddzq9Q5+wPmaXJSIiF/Hvv/D335CWdmkh5uzneXlX9/779xdPP66UApBcEcMwmLx6MsPih5FnzaNBjQb8t/t/aezX2OzSRETkLIcPw4YN9svu3cWz7ypVChZv74LlfI/Pfl6nTvG8/5VSAJLL9u/Jf4n5OoavE78G4KHGDzH7ntlUca1icmUiIpWbYcDBg+eGnfONttSqBTVqXFpgKeqxlxc4lNML6igAyWVZf3A9D37+ILuP7cbF0YW3OrzFEy2ewGKxmF2aiEilYhiwZ8+5YSc5+dy2Fgtcdx3ceGPB0qIFhIVBtWqlXnaZoQAkl8QwDGasm8GzS58lJz+HOlXr8N/u/+XGwBvNLk1EpMKzWmHnznPDztGj57Z1cIAbbrAPO82aFYzayGkKQHJRx7OP8/i3j/Pplk8B6NqwK3PvnUtVt6rmFiYiUgHl50Nion3Q2bixYOLx2ZydoXHj00HnxhuhSRPw0Em4F6UAJBf0R/IfPPD5A2w7sg0nBycmRk5kcOvBOuQlInIF8vMLzqQ688yr9HQ4dKgg5KxfD7//DpmZ527r6lowknNm2GnUqGC9XD4FIDmvuRvn8vTipzmZd5JrvK/hswc+o01IG7PLEhEpdYXBpTCwnB1gLvV5RsalvZ+nZ8EcncKgc+ON0LBhwYhPmWfNg+wUyEqGk8mQlVTwOCsZTp7xuP6TcN1TppWpACTnyMzN5OnFTzNv0zwAOtTrwIf3fYivh6+5hYmIFDPDgH37CkZe1q0rOPR0dnhJT4cTJ4r3fd3c7M+2ql4dmjY9HXjq1wdHx+J9z6tizYPs1IIwc/JUgDlfsMlOBYyL7/PEzhIv+0IUgMROYmoiD3z+AFsOb8HB4sCY28cw7JZhOFjK6XmOIiKnGAYcOHA67BT+TEm59H24uoKPj3148fY+d93Fnru4lFw/L5kt1BQRYs4ON5caagpZHMC1Jrj5g1tAwU/3Mx67+YO3ubdKUgASm0+3fEq/b/pxIucE/p7+fNLtE26vc7vZZYmIXJFDhwoCzplhp6hTxJ2cCiYSh4cXTCA+87o4Z4aXKlXK+HybvJMFh56yUyArpSC02B4X8Tzn38vbv12oORVm3P3tQ07h4uoLDmVpCOtcCkBCdl42zy59lhnrZgDQLrQdn3T7hACvAJMrExG5NMnJ9kFn3bqCAHQ2R8eCicPh4QWHm8LDCw49ubmVfs0XZBiQm3ZqhCblIsHm1Lq8S5xgZMcCbjXPDTDuAfZBp5yEmsuhAFTJ7fp3F90/7876Q+sBeOnWlxjVbhRODvqrISJlU0qKfdBZv77oKx0XXg+nMOiEhxecReXuXvo1A2dNDk46NZ/mjJ+FwaYw0FhzL/89HJwLRmlcaxYEG9eaBcHl7OeFj12qV6hQczn0LVeJff3310QvjCYtO40a7jX46P6P6FCvg9lliYjYHDly7pydvXvPbWexFJwlVRh0Cq907OlZwgUaBuQeKzrQnPkzK6lgxOZy5tEAOHmdG1rOF2zcaoJTlYJfhlyUAlAllJufy7AfhjH518kAtLmmDQseWECIT4jJlYlIRWYYkJVVcJZV4ZlWhY/Pfr5vX0HY+eefovfVoIH9yE5YWDFf6Tgv88KBxhZsksGac+n7tTiAq9+pQ0wBp3+6+Z8RZs4INo5l7dhcxaEAVMnsS9tHj//2YPX+1QA81+Y5xt05DmfH8nBxCRExi9V6+jo45wstF3ueng65V3BUp149+5Gd5s0LJiZfEsMKuemQcxSyj0L2kYLHOWc8zj7jeeFZUXnHL69I56rnhhpbuDnjcQWbR1OeKQBVIkt2LOHRLx/lyMkj+Lj6MK/rPLo27Gp2WSJSRlitsH17waGmwmXXroLwcvx4wQhOcbBYCkZrfHxOL4VnWxU+9vM7fQHAqlU5K8gcgYNnhRhbkDlyOtAULob1ygp1dAO3wEsINn4aqSmHFIAqgdz8XF5Z8Qqv/vwqBgY3Bt7I5w9+Tt1qdc0uTURMYrXCtm32YWfjxoKgcyHOzvbBpajwcrHXvDytOOT9e2rSb+oZZzalnnG6dmpBkPn5VKjJ+ffKgwyAowe4Vi+Y9Otao+CnS/VT62rYv1YYbDSfpkJTAKrgVvyzgqcXP82fKX8C8GT4k0yOmoybk/63IlJZnHlzzTPDTlFXN3Z3P30LhhYtCs6iqlbtdHgp8nTxvMyLhJkUOJQK/5x6PefIVYzKeJwOMJcSaArXaYRGzqIAVEEdPH6Q579/nk+2fAJADfcaTLt7Gg81fsjkykSkJOXnw99/24/sbNpU9D2oPDzsw06LFtCwgRUn67/nXmMmJQX2nzVCUxh28ou4c+elcPY+42wmX/sJwK41zggyhT+rKchIsVEAqmBy8nN469e3eOWnVziRcwILFp5o8QSv3vkq1d2rm12eiBSjvLyiw86ZdxK3WKxU9zzKdXVTaHNjCi0apXDDtYepG5SCb5UUHHJSIPtwQZjZlgJ/pIKRf/nF2K4/43tGiDkz1Pie9bovOJaF+0FIZaUAVIHE74pnwP8G8Hfq3wBEBEcw/e7ptAhqYXJlInK18vJg61bYsD6fv38/yp5tKaQeSMHbNQU/78PU9E6hZ/0UnmmRQmC1w1xTM4WaVVLwdDqCg6WIQJN8ajkfZ59zrz9zzijNGSM3mi8j5YwCUAWwL20fz33/HJ//9TkANT1qMiFyAtFh0bqJqUgZkpdXcCr4sWOnTg0/ZpCRdoKstFRyM1Kxniw4tOSYm4pT/hFcLal4OKTi6ZxKFecU/L1SeLTKERxbWqHlZb65c9UzLpjnd9YF9M567uoLjmX5plciV69MBKDp06fz+uuvk5SURLNmzXj77bdp1arVedt//vnnjBw5kn/++Yf69eszYcIE7r77btvrffr0Yf78+XbbREVFsWTJkhLrgxmy87J589c3GfPTGDJzM3GwOPBU+FO8cvsrVHOvZnZ5IhVOTk7B/aXOvM6NLcykGWQezyQ/oyDEOOSm4mQ9gquRirtDKl7Oqfi4peJbpWDx8zrCDVVScXXOAWeg6qnlEmUb1cDND5cqNbG4XSDMuPmduvaMrvUlcibTA9CCBQuIjY1l5syZREREMGXKFKKiokhMTMTPz++c9qtWraJnz56MGzeOe+65h48//piuXbuyYcMGGjdubGvXoUMH5s6da3vuWqZv4Xv5vt/5PQP/N5BtR7YBcHPIzUy7exphAWHmFiZSQeTmwpYtp2+/8OfvJ7jG4X80CPjTFmJ8q6TSyOtIweOgVNxdsq7ovbJyPTiRW4PMfF+yDF9yHHzJd/LFcPHFwd0XZ88aePn6ElzXDwf3ggnCrgo0IlfFYhjFdWmrKxMREUHLli2ZNm0aAFarlZCQEAYOHMiwYcPOad+jRw8yMjL49ttvbetat25NWFgYM2fOBApGgI4dO8bChQuvqKb09HR8fHxIS0vD29v7ivZRUvYc20Ps97F8ufVLAPw9/Zl410R6Ne2FRcffRa5IXh789Zf9zTV//x3cHI/R+cZv6NbyC6KaLr2kgJNndeWk4UuuxZe8UyHG4uaLk2cNXKr44ubji5PHGROBXWuAk0cp9FKk4ruc729TR4BycnJYv349w4cPt61zcHAgMjKS1atXF7nN6tWriY2NtVsXFRV1TthJSEjAz8+PatWqcccddzB27Fhq1KhR5D6zs7PJzs62PU9PT7/CHpWcrLwsJq2axGs/v8bJvJM4WhwZ2Gogo9uNxsftUq8JLyKFZ06deXPNTZsK7lEFUMMrlXtbfM3oQf/lzkbxuDidvndDrls9nIJvx+Lmb382k5vvqVO1fXFy8qSK/jMiUuaZGoBSU1PJz8/H39/fbr2/vz9///13kdskJSUV2T4pKcn2vEOHDtx///3UqVOHnTt38uKLL9KxY0dWr16No+O592AZN24cL7/8cjH0qGQs3r6YZ/73DDv/3QnAbbVvY1rHaTTxb2JyZSJlW35+wdWOzxzZ2bjR/jRxgICqh3g48iseue0LmgWtwPHMs6Z8boCQbhDyAM5Vm+hMJ5EKwvQ5QCXhoYdOX+yvSZMmNG3alGuvvZaEhATuvPPOc9oPHz7cblQpPT2dkBDz74y++9/dDF46mEWJiwAI9ApkUvtJ9GzcU4e7RM5SeB+rM0d2Nmwo+gKAXl7Q8ba9PHzbl9wU8gU1WYmFM2YDVAuDkAcKgo9Pw1Lrg4iUHlMDkK+vL46OjiQn21+MIjk5mYCAgCK3CQgIuKz2AHXr1sXX15cdO3YUGYBcXV3L1CTpk7knmbhyIuNXjicrLwsnBycGRQwirm0c3q5la06SiBms1oKbdBaO7BSGnaLuY+XpWXD38PBwaBe+gzbBX1Az6wssR3+zb1gjoiDw1OoGXrpPnkhFZ2oAcnFxoUWLFsTHx9O1a1egYBJ0fHw8AwYMKHKbNm3aEB8fz+DBg23rli1bRps2bc77Pvv37+fIkSMEBgYWZ/kl4pvEbxi0ZBC7j+0G4PbQ25l29zRuqHmDyZWJXB3DgJMnC0JKenrBUvj4UtcVPj7fncnd3U+HnRYtCn42CPgLxwNfwL4v4NjvcLCwtQVq3nLq8Nb94Gn+qK+IlB7TD4HFxsYSHR1NeHg4rVq1YsqUKWRkZBATEwNA7969CQ4OZty4cQAMGjSItm3b8sYbb9CpUyc+/fRT1q1bx+zZswE4ceIEL7/8Mt26dSMgIICdO3cyZMgQ6tWrR1RUlGn9vJgdR3cweMlgvtv+HQDBVYKZHDWZB294UIe7pMwpvJP4hg1w8OClh5j8K7jDwvm4uUGzZgUhp3Bp2BCcHI2CoLP3v7DrC9h0xnxCiyP4tYNaD8A1XQvu+C0ilZLpAahHjx6kpKQQFxdHUlISYWFhLFmyxDbRee/evTg4nL6a8U033cTHH3/MSy+9xIsvvkj9+vVZuHCh7RpAjo6ObN68mfnz53Ps2DGCgoJo3749Y8aMKVOHuQpl5mYy7udxTFw1kZz8HJwdnIltE8tLt72El4uX2eWJ2CYSn3m/qfPdSfxSWCxQpUrB4u19+uelPD5zXfXq4Fx4KRzDgCNr4Y9TIz0ndp1+QwdnCLirYKQnuEvBGVsiUumZfh2gsqg0rgNkGAYL/17Is0ufZU/aHgDuqnsXb3d8mwa+DUrkPUUu5syzps4MO0VNJHZ3L7iTeN264ONz/qBy9mNPT3Aojju0WPMhdVVB4Nn3JWTuO/2aoxsEdjgVeu4Bl6rF8IYiUtaVm+sAVVbbjmzjmf89w9KdSwEI8Q7hzag3uf/6+3W4S0pNfj4kJp47slNU2PHwKAg7LVqcXho2BKeS/AQxDMg7DpkHIevQ6Z8nD0HmATicAFlnnBDh5AlBnQoObwV2BGeNoIrI+SkAlaKMnAzG/jSWN1a/Qa41FxdHF1646QWG3zIcTxdPs8uTCiw/v+Dif2eGnU2bzh92mjc/N+wUcQmtK2MYkHusIMicPHjq53ke52deeF/OPgWHtWp1g4D24OReTEWKSEWnAFSK+n/Xn482fwRAx3odeavDW9SvUd/kqqSiKbzS8dlh5+yL/4F92Ck8c6pBgysMO4YBOUfPCjNFhJqsQ5B/GffMcvYG90BwDwK3QPA49dPnBvC/AxxdrqBYEansFIBK0YhbR7D2wFomRk6kS4MuOtwll8wwCm7VcL4zrNLTYevW07d1OHny3H0UXg/nzJGdSw471vyC4JKxr2CujW3ZX3A4qvDQlDXn0jvlXPV0mCkMOHY/Ty1OGh0VkeKnAFSKGvo2ZOvTW3GwFMcMUCkPcnIufD2bi13v5szHeXmX/r5eXueGneuuO0/YMQzIOnxWsNlnH3ZOHgTjEs9hd6l+VogpIti4BepwlYiYSgGolCn8VCxn3kW8cBJxcvLp8HLGPXaLjZdX0aeH16lzOuzUr38q7BgG5PxbEGKSigg2haM4lzJyY3EsCDEeIQWL56mf7sGn1geBWwA4lr3LTYiInE0BSOQS5eWdPsx05tyarEuYzuLufnXXvCl87OV16hTy/GzIOVYQbgqXkwcLAs2xfbDijLBzsYnEAFgKLgpYGG7ODjkeIQXhxqG4ZkKLiJhLAUikCIVhpzDorFsHv/9e9NyaKlXsDzXVrn1ukCnydPG8k/YBJuffgrOjzl53+F/Yf9a6/CIKuRBX3/MHG4+QghEcTSYWkUpEAUgqvbPPmrrQRGIvL/uwEx4O9eqBA/mQuQfSEwuuTZPzL6T9CynHzg00hYv1ao+PWQpOA3epVrC4+Z8bbDxCwOMazbcRETmLApBUKmdeD6fwUNb5ThH38oIbb7QPO/XrZOJwIhHS/y5YDm+FHX9D+rYrCzQWh4Lw4lyt4GrFhWHmUhZn74LtRUTksikASYV15pWOz5ykXFTY8fQ8HXbCww1aNT3MtTX/xuH435C+tSDs/P03rN9z/jd0cIUq9QtGXGxBpeqFQ4xTlYKbY4mISKlSAJJy7fhx2LsX9uw5vezdC7t3wx9/nP9KxzfeCC1b5NE2fDc31ttKsNffOJw4NaqTthX+OHb+N3WpDj7Xg/f14N2wYPG5Hjxqa5KwiEg5oQAkZZZhwOHDp0PN2SFnzx74998L78PDA25qeYKom/6m9Q1/0yDwb3yd/8ZyfCsc3w7WXNhV1JYW8KpzOuB4NzwdeHQ3cRGRck8BSEyTmwv7958/4Ozde/FTzF2csrkuJJlm1yXRMDSJa4OTCamZRIDPIQI8t+OZ/zeWk/tPb5B81g4c3cG7gX3A8W5YcChLE4dFRCosBSApMVlZsGsX/PNP0SHn4MGCUZ6zOTnmUrNKCtcHJBHgk0yD2klcVyuJUP9kgqol4VslCR+XZNxJwtF6rOg3N4ATZzx38zs35Hg3BM9amkgsIlIJKQDJVTl5EnbuhB07Cpbt20//3L//dMCxWKz4VknF3yeZAJ8kbq+ThH+zZIJrJFE3MIlrfJPx90mimnsSHg5HsFiKSEZnsp7x2MG54CJ9bv4FP91PPfaqeyrwNADX6iX2OxARkfJHAUguKjOzYCTnzHBzZsgBg5reKdSqsZfavnto5ruHLu32UKvGXur47yWo+iFqeBzG0eES7yUFBaMyrn6nwsypQHO+xy7VdCaViIhcFgUgAQpCzs6dRYecpEN5XFN9P7V8CwJObd89tLx2D7VaFTyv5bsXD5dLvDKxq+/FA417ALjU0BlVIiJSYhSAKpGMjKJDzv5/MnDOPR1uavvuoW2NvfS+dQ+179tDcPUDODpYL/4G7oEFp4J71i6YW+NZGzxqgUfwqXBTs+BwlYiIiMkUgCqYM0dytm+HnTvySN1/mJNHD+JuPR1y6vvuJbLOHmq33INvlSMX3a/h4ILFI+RUuDkVbAofe9YuuPif7gIuIiLlhAJQOZSVBbu2Z7NvexIpew+RfvgQOWmHsGQdwsvxIIFVD1Gn6iFuqnoIv7aHcXC4yIRiwOrkjYNX7XNHcE4tFjd/nS0lIiIVhgJQWZOXCScPkZN+iMN7DnLs0CEyjxzCmnEIp7xDeDkewtfzEDdUOcINAH6nlguwGo7kOfnj4BWCk0/tIkdxHFx8SqFzIiIiZYMCUGk6eajgVgsnD8HJQ+SfOETGkUPkph3Ekn0Idw7h7pQOgAtwzakFr1PLWXLyXEjLCSDLEojhFoSLTyBV/ALxqB6IxT2wYE6OexAOrr64aEKxiIiIjQJQKdq1dAZ1M8fYnjsC3oVPzviTyMx25+C/QRw+HsgJayB5ToE4egXiUSOQakFBBNYNpHpQIC6u1amp079FREQumwJQKfpzb11ychpw6FigbUk9EYThFohr1UB8AgKpWSuQ0Hre1L/OwrX+uryNiIhISVAAKkU1wvsw9+s+1K8P9etDu/oQGKiQIyIiUtoUgErRTTcVLCIiImIundcsIiIilY4CkIiIiFQ6CkAiIiJS6ZSJADR9+nRCQ0Nxc3MjIiKCtWvXXrD9559/TsOGDXFzc6NJkyYsXrzY7nXDMIiLiyMwMBB3d3ciIyPZvn17SXZBREREyhHTA9CCBQuIjY1l1KhRbNiwgWbNmhEVFcXhw4eLbL9q1Sp69uxJ37592bhxI127dqVr165s2bLF1mbixIlMnTqVmTNnsmbNGjw9PYmKiiIrK6u0uiUiIiJlmMUwjIvfKKoERURE0LJlS6ZNmwaA1WolJCSEgQMHMmzYsHPa9+jRg4yMDL799lvbutatWxMWFsbMmTMxDIOgoCCee+45nn/+eQDS0tLw9/dn3rx5PPTQQxetKT09HR8fH9LS0vD29r5o+0tmGAV3KxUREansPDyK/Towl/P9bepp8Dk5Oaxfv57hw4fb1jk4OBAZGcnq1auL3Gb16tXExsbarYuKimLhwoUA7N69m6SkJCIjI22v+/j4EBERwerVq4sMQNnZ2WRnZ9uep6enX023zi8zE7yKuKeFiIhIZXPiBHh6mvb2ph4CS01NJT8/H39/f7v1/v7+JCUlFblNUlLSBdsX/rycfY4bNw4fHx/bEhISckX9ERERkfJBF0IEhg8fbjeqlJ6eXjIhyMOjIPGKiIhUdh4epr69qQHI19cXR0dHkpOT7dYnJycTEBBQ5DYBAQEXbF/4Mzk5mcDAQLs2YWFhRe7T1dUVV1fXK+3GpbNYTB3uExERkQKmHgJzcXGhRYsWxMfH29ZZrVbi4+Np06ZNkdu0adPGrj3AsmXLbO3r1KlDQECAXZv09HTWrFlz3n2KiIhI5WL6IbDY2Fiio6MJDw+nVatWTJkyhYyMDGJiYgDo3bs3wcHBjBs3DoBBgwbRtm1b3njjDTp16sSnn37KunXrmD17NgAWi4XBgwczduxY6tevT506dRg5ciRBQUF07drVrG6KiIhIGWJ6AOrRowcpKSnExcWRlJREWFgYS5YssU1i3rt3Lw4OpweqbrrpJj7++GNeeuklXnzxRerXr8/ChQtp3Lixrc2QIUPIyMjg8ccf59ixY9xyyy0sWbIENze3Uu+fiIiIlD2mXweoLCqx6wCJiIhIibmc72/TrwQtIiIiUtoUgERERKTSUQASERGRSkcBSERERCodBSARERGpdBSAREREpNJRABIREZFKRwFIREREKh0FIBEREal0TL8VRllUeHHs9PR0kysRERGRS1X4vX0pN7lQACrC8ePHAQgJCTG5EhEREblcx48fx8fH54JtdC+wIlitVg4ePEiVKlWwWCxml3NV0tPTCQkJYd++fRX+vmbqa8VTWfoJ6mtFVFn6CWWnr4ZhcPz4cYKCguxupF4UjQAVwcHBgWuuucbsMoqVt7d3hf8HWEh9rXgqSz9Bfa2IKks/oWz09WIjP4U0CVpEREQqHQUgERERqXQUgCo4V1dXRo0ahaurq9mllDj1teKpLP0E9bUiqiz9hPLZV02CFhERkUpHI0AiIiJS6SgAiYiISKWjACQiIiKVjgKQiIiIVDoKQOXAuHHjaNmyJVWqVMHPz4+uXbuSmJho1yYrK4unn36aGjVq4OXlRbdu3UhOTrZrs3fvXjp16oSHhwd+fn688MIL5OXl2bVJSEjgxhtvxNXVlXr16jFv3ryS7t55jR8/HovFwuDBg23rKlI/Dxw4wKOPPkqNGjVwd3enSZMmrFu3zva6YRjExcURGBiIu7s7kZGRbN++3W4fR48e5ZFHHsHb25uqVavSt29fTpw4Yddm8+bN3Hrrrbi5uRESEsLEiRNLpX+F8vPzGTlyJHXq1MHd3Z1rr72WMWPG2N2rp7z29aeffqJz584EBQVhsVhYuHCh3eul2a/PP/+chg0b4ubmRpMmTVi8eHGp9DM3N5ehQ4fSpEkTPD09CQoKonfv3hw8eLDc9fNifT1b//79sVgsTJkyxW59Rerr1q1b6dKlCz4+Pnh6etKyZUv27t1re71cfyYbUuZFRUUZc+fONbZs2WJs2rTJuPvuu41atWoZJ06csLXp37+/ERISYsTHxxvr1q0zWrdubdx000221/Py8ozGjRsbkZGRxsaNG43Fixcbvr6+xvDhw21tdu3aZXh4eBixsbHGX3/9Zbz99tuGo6OjsWTJklLtr2EYxtq1a43Q0FCjadOmxqBBg2zrK0o/jx49atSuXdvo06ePsWbNGmPXrl3G0qVLjR07dtjajB8/3vDx8TEWLlxo/P7770aXLl2MOnXqGCdPnrS16dChg9GsWTPj119/NX7++WejXr16Rs+ePW2vp6WlGf7+/sYjjzxibNmyxfjkk08Md3d3Y9asWaXW11dffdWoUaOG8e233xq7d+82Pv/8c8PLy8t46623yn1fFy9ebIwYMcL48ssvDcD46quv7F4vrX6tXLnScHR0NCZOnGj89ddfxksvvWQ4Ozsbf/zxR4n389ixY0ZkZKSxYMEC4++//zZWr15ttGrVymjRooXdPspDPy/W1zN9+eWXRrNmzYygoCDjzTffrJB93bFjh1G9enXjhRdeMDZs2GDs2LHD+Prrr43k5GRbm/L8mawAVA4dPnzYAIwVK1YYhlHwAeTs7Gx8/vnntjZbt241AGP16tWGYRT8RXdwcDCSkpJsbWbMmGF4e3sb2dnZhmEYxpAhQ4xGjRrZvVePHj2MqKioku6SnePHjxv169c3li1bZrRt29YWgCpSP4cOHWrccsst533darUaAQEBxuuvv25bd+zYMcPV1dX45JNPDMMwjL/++ssAjN9++83W5n//+59hsViMAwcOGIZhGO+8845RrVo1W98L37tBgwbF3aXz6tSpk/HYY4/Zrbv//vuNRx55xDCMitPXs79ASrNf3bt3Nzp16mRXT0REhPHEE08Uax8N49x+FmXt2rUGYOzZs8cwjPLZT8M4f1/3799vBAcHG1u2bDFq165tF4AqUl979OhhPProo+fdprx/JusQWDmUlpYGQPXq1QFYv349ubm5REZG2to0bNiQWrVqsXr1agBWr15NkyZN8Pf3t7WJiooiPT2dP//809bmzH0UtincR2l5+umn6dSp0zm1VKR+Llq0iPDwcB588EH8/Pxo3rw57777ru313bt3k5SUZFenj48PERERdn2tWrUq4eHhtjaRkZE4ODiwZs0aW5vbbrsNFxcXW5uoqCgSExP5999/S7qbANx0003Ex8ezbds2AH7//Xd++eUXOnbsCFSsvp6pNPtVFv5OnyktLQ2LxULVqlWBitVPq9VKr169eOGFF2jUqNE5r1eUvlqtVr777juuu+46oqKi8PPzIyIiwu4wWXn/TFYAKmesViuDBw/m5ptvpnHjxgAkJSXh4uJi+7Ap5O/vT1JSkq3NmX8BC18vfO1CbdLT0zl58mRJdOccn376KRs2bGDcuHHnvFaR+rlr1y5mzJhB/fr1Wbp0KU8++STPPPMM8+fPt6u1qDrP7Iefn5/d605OTlSvXv2yfh8lbdiwYTz00EM0bNgQZ2dnmjdvzuDBg3nkkUfs6qgIfT1TafbrfG3M6HdWVhZDhw6lZ8+etptiVqR+TpgwAScnJ5555pkiX68ofT18+DAnTpxg/PjxdOjQge+//5777ruP+++/nxUrVthqLM+fybobfDnz9NNPs2XLFn755RezSyl2+/btY9CgQSxbtgw3NzezyylRVquV8PBwXnvtNQCaN2/Oli1bmDlzJtHR0SZXV7w+++wz/u///o+PP/6YRo0asWnTJgYPHkxQUFCF62tll5ubS/fu3TEMgxkzZphdTrFbv349b731Fhs2bMBisZhdTomyWq0A3HvvvTz77LMAhIWFsWrVKmbOnEnbtm3NLK9YaASoHBkwYADffvsty5cv55prrrGtDwgIICcnh2PHjtm1T05OJiAgwNbm7Jn5hc8v1sbb2xt3d/fi7s451q9fz+HDh7nxxhtxcnLCycmJFStWMHXqVJycnPD3968Q/QQIDAzkhhtusFt3/fXX286uKKy1qDrP7Mfhw4ftXs/Ly+Po0aOX9fsoaS+88IJtFKhJkyb06tWLZ5991jbKV5H6eqbS7Nf52pRmvwvDz549e1i2bJlt9KewvorQz59//pnDhw9Tq1Yt22fUnj17eO655wgNDbXVWBH66uvri5OT00U/p8rzZ7ICUDlgGAYDBgzgq6++4scff6ROnTp2r7do0QJnZ2fi4+Nt6xITE9m7dy9t2rQBoE2bNvzxxx92/zALP6QK/4K3adPGbh+FbQr3UdLuvPNO/vjjDzZt2mRbwsPDeeSRR2yPK0I/AW6++eZzLmWwbds2ateuDUCdOnUICAiwqzM9PZ01a9bY9fXYsWOsX7/e1ubHH3/EarUSERFha/PTTz+Rm5tra7Ns2TIaNGhAtWrVSqx/Z8rMzMTBwf6jxtHR0fY/zIrU1zOVZr/M/jtdGH62b9/ODz/8QI0aNexeryj97NWrF5s3b7b7jAoKCuKFF15g6dKlthorQl9dXFxo2bLlBT+nyv13T4lOsZZi8eSTTxo+Pj5GQkKCcejQIduSmZlpa9O/f3+jVq1axo8//misW7fOaNOmjdGmTRvb64WnIrZv397YtGmTsWTJEqNmzZpFnor4wgsvGFu3bjWmT59u2mnwhc48C8wwKk4/165dazg5ORmvvvqqsX37duP//u//DA8PD+Ojjz6ytRk/frxRtWpV4+uvvzY2b95s3HvvvUWeQt28eXNjzZo1xi+//GLUr1/f7nTbY8eOGf7+/kavXr2MLVu2GJ9++qnh4eFRqqfBR0dHG8HBwbbT4L/88kvD19fXGDJkSLnv6/Hjx42NGzcaGzduNABj8uTJxsaNG21nP5VWv1auXGk4OTkZkyZNMrZu3WqMGjWqWE+ZvlA/c3JyjC5duhjXXHONsWnTJrvPqDPPcioP/bxYX4ty9llgFamvX375peHs7GzMnj3b2L59u+309J9//tm2j/L8mawAVA4ARS5z5861tTl58qTx1FNPGdWqVTM8PDyM++67zzh06JDdfv755x+jY8eOhru7u+Hr62s899xzRm5url2b5cuXG2FhYYaLi4tRt25du/cww9kBqCL185tvvjEaN25suLq6Gg0bNjRmz55t97rVajVGjhxp+Pv7G66ursadd95pJCYm2rU5cuSI0bNnT8PLy8vw9vY2YmJijOPHj9u1+f33341bbrnFcHV1NYKDg43x48eXeN/OlJ6ebgwaNMioVauW4ebmZtStW9cYMWKE3Zdjee3r8uXLi/y3GR0dXer9+uyzz4zrrrvOcHFxMRo1amR89913pdLP3bt3n/czavny5eWqnxfra1GKCkAVqa/vv/++Ua9ePcPNzc1o1qyZsXDhQrt9lOfPZIthnHE5VhEREZFKQHOAREREpNJRABIREZFKRwFIREREKh0FIBEREal0FIBERESk0lEAEhERkUpHAUhEREQqHQUgERERqXQUgESkwmrXrh2DBw8GIDQ0lClTpphaj4iUHU5mFyAiUhp+++03PD09zS5DRMoIBSARqRRq1qxpdgkiUoboEJiIVAgZGRn07t0bLy8vAgMDeeONN+xeP/sQmMViYdasWdxzzz14eHhw/fXXs3r1anbs2EG7du3w9PTkpptuYufOnaXcExEpDQpAIlIhvPDCC6xYsYKvv/6a77//noSEBDZs2HDBbcaMGUPv3r3ZtGkTDRs25OGHH+aJJ55g+PDhrFu3DsMwGDBgQCn1QERKkw6BiUi5d+LECd5//30++ugj7rzzTgDmz5/PNddcc8HtYmJi6N69OwBDhw6lTZs2jBw5kqioKAAGDRpETExMyRYvIqbQCJCIlHs7d+4kJyeHiIgI27rq1avToEGDC27XtGlT22N/f38AmjRpYrcuKyuL9PT0Yq5YRMymACQilZazs7PtscViOe86q9VauoWJSIlTABKRcu/aa6/F2dmZNWvW2Nb9+++/bNu2zcSqRKQs0xwgESn3vLy86Nu3Ly+88AI1atTAz8+PESNG4OCg/+OJSNEUgESkQnj99dc5ceIEnTt3pkqVKjz33HOkpaWZXZaIlFEWwzAMs4sQERERKU0aHxYREZFKRwFIREREKh0FIBEREal0FIBERESk0lEAEhERkUpHAUhEREQqHQUgERERqXQUgERERKTSUQASERGRSkcBSERERCodBSARERGpdP4fHn4ARtWm8kkAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "forward:\n",
      "        dim  Triton-DYT  Torch-DYT   RMSNorm  Liger-DYT\n",
      "0    1024.0    0.011484   0.038008  0.012312        0.0\n",
      "1    2048.0    0.016751   0.059825  0.015905        0.0\n",
      "2    3072.0    0.021949   0.087552  0.020293        0.0\n",
      "3    4096.0    0.026335   0.116589  0.023672        0.0\n",
      "4    5120.0    0.033668   0.141492  0.029418        0.0\n",
      "5    6144.0    0.035613   0.165582  0.031831        0.0\n",
      "6    7168.0    0.042901   0.190179  0.035965        0.0\n",
      "7    8192.0    0.044846   0.214588  0.039585        0.0\n",
      "8    9216.0    0.052222   0.237241  0.051461        0.0\n",
      "9   10240.0    0.054397   0.264036  0.054295        0.0\n",
      "10  11264.0    0.061670   0.288258  0.057325        0.0\n",
      "11  12288.0    0.063772   0.313241  0.060236        0.0\n",
      "12  13312.0    0.071145   0.335191  0.062842        0.0\n",
      "13  14336.0    0.073178   0.361316  0.066260        0.0\n",
      "14  15360.0    0.080337   0.386052  0.069228        0.0\n",
      "15  16384.0    0.082294   0.408363  0.072553        0.0\n"
     ]
    }
   ],
   "source": [
    "\n",
    "@triton.testing.perf_report(\n",
    "    triton.testing.Benchmark(\n",
    "        x_names=['dim'],  # argument names to use as an x-axis for the plot\n",
    "        x_vals=[1024 * i for i in range(1, 16+1)],  # different possible values for `x_name`\n",
    "        line_arg='provider',  # argument name whose value corresponds to a different line in the plot\n",
    "        line_vals=['Triton-DYT', \"Torch-DYT\", 'RMSNorm', \"Liger-DYT\"],  # possible values for `line_arg``\n",
    "        line_names=[\n",
    "            \"Triton-DYT\",\n",
    "            \"Torch-DYT\",\n",
    "            \"RMSNorm\",\n",
    "            \"Liger-DYT\"\n",
    "        ],  # label name for the lines\n",
    "        styles=[('blue', '-'), ('green', '-'), ('orange', '-'), ('red', '-')],  # line styles\n",
    "        ylabel=\"ms\",  # label name for the y-axis\n",
    "        plot_name=\"forward\",  # name for the plot. Used also as a file name for saving the plot.\n",
    "        args={'seq_len': 1024, 'bs': 4, 'beta': False}\n",
    "    ))\n",
    "def benchmark(bs, seq_len, dim, beta, provider):\n",
    "    device = torch.device('cuda')\n",
    "    dtype = torch.bfloat16\n",
    "    tensor = torch.randn(bs, seq_len, dim).to(device).to(dtype)\n",
    "    stream = torch.cuda.Stream()\n",
    "    torch.cuda.set_stream(stream)\n",
    "    if provider == 'Triton-DYT':\n",
    "        dyt = DYT(dim, beta=beta).to(device).to(dtype)\n",
    "        ms = triton.testing.do_bench(lambda: dyt(tensor, 'triton'))\n",
    "    if provider == 'Torch-DYT':\n",
    "        dyt = DYT(dim, beta=beta).to(device).to(dtype)\n",
    "        ms = triton.testing.do_bench(lambda: dyt(tensor, 'torch'))\n",
    "    if provider == 'RMSNorm':\n",
    "        norm = TritonRMSNorm(dim).to(device).to(dtype)\n",
    "        # norm = FusedRMSNorm(dim).to(device).to(dtype)\n",
    "        # norm = TERMSNorm(dim).to(device).to(dtype)\n",
    "        ms = triton.testing.do_bench(lambda: norm(tensor))\n",
    "    if provider == 'Liger-DYT':\n",
    "        if not beta:\n",
    "            return 0\n",
    "        dyt = LigerDyT(dim).to(device).to(dtype)\n",
    "        ms = triton.testing.do_bench(lambda: dyt(tensor))\n",
    "    return ms\n",
    "benchmark.run(show_plots=True, print_data=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "@triton.testing.perf_report(\n",
    "    triton.testing.Benchmark(\n",
    "        x_names=['dim'],  # argument names to use as an x-axis for the plot\n",
    "        x_vals=[1024 * i for i in range(1, 16+1)],  # different possible values for `x_name`\n",
    "        line_arg='provider',  # argument name whose value corresponds to a different line in the plot\n",
    "        line_vals=['Triton-DYT-2sum', \"Triton-DYT-1sum\"],  # possible values for `line_arg``\n",
    "        line_names=[\n",
    "            \"Triton-DYT-2sum\",\n",
    "            \"Triton-DYT-1sum\"\n",
    "        ],  # label name for the lines\n",
    "        styles=[('blue', '-'), ('green', '-'), ('orange', '-'), ('red', '-')],  # line styles\n",
    "        ylabel=\"ms\",  # label name for the y-axis\n",
    "        plot_name=\"backward\",  # name for the plot. Used also as a file name for saving the plot.\n",
    "        args={'seq_len': 1024, 'bs': 4, 'beta':True}\n",
    "    ))\n",
    "def benchmark(bs, seq_len, dim, beta, provider):\n",
    "    device = torch.device('cuda')\n",
    "    dtype = torch.bfloat16\n",
    "    tensor = torch.randn(bs, seq_len, dim).to(device).to(dtype)\n",
    "    tensor.requires_grad_(True)\n",
    "    dy = torch.rand_like(tensor)\n",
    "    stream = torch.cuda.Stream()\n",
    "    torch.cuda.set_stream(stream)\n",
    "    if provider == 'Triton-DYT-2sum':\n",
    "        dyt = DYT(dim, beta=beta).to(device).to(dtype)\n",
    "        y = dyt(tensor, 'triton')\n",
    "        ms = triton.testing.do_bench(lambda: y.backward(dy, retain_graph=True), grad_to_none=[tensor])\n",
    "    if provider == 'Liger-DYT-1sum':\n",
    "        dyt = DYT2(dim, beta=beta).to(device).to(dtype)\n",
    "        y = dyt(tensor, 'triton')\n",
    "        ms = triton.testing.do_bench(lambda: y.backward(dy, retain_graph=True), grad_to_none=[tensor])\n",
    "        # if not beta:\n",
    "        #     return 0\n",
    "        # dyt = LigerDyT(dim).to(device).to(dtype)\n",
    "        # y = dyt(tensor)\n",
    "        # ms = triton.testing.do_bench(lambda: y.backward(dy, retain_graph=True), grad_to_none=[tensor])\n",
    "    return ms\n",
    "benchmark.run(show_plots=True, print_data=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkAAAAGwCAYAAABB4NqyAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAaPlJREFUeJzt3Xl8TFfjBvBnJvseEVmFxK6EqBBRpSUVqlSrpCihXi0tpam1CH3VWlqKn6Vva2mrVW9VW+1raQhFSi2xN/ZdElsy2be5vz+OzGSykW3uLM/387mf3Ny5c+echJkn555FIUmSBCIiIiIzopS7AERERET6xgBEREREZocBiIiIiMwOAxARERGZHQYgIiIiMjsMQERERGR2GICIiIjI7FjKXQBDpFarcfv2bTg5OUGhUMhdHCIiInoCkiQhLS0NPj4+UCrLb+NhACrF7du34efnJ3cxiIiIqBJu3LiBunXrlnsOA1ApnJycAIgfoLOzs8ylISIioiehUqng5+en+RwvDwNQKQpvezk7OzMAERERGZkn6b7CTtBERERkdhiAiIiIyOwwABEREZHZYR+gKigoKEBeXp7cxaBqZGVlBQsLC7mLQURENYwBqBIkSUJiYiJSUlLkLgrVAFdXV3h5eXEOKCIiE8YAVAmF4cfDwwP29vb8oDQRkiQhMzMTycnJAABvb2+ZS0RERDWFAaiCCgoKNOGndu3acheHqpmdnR0AIDk5GR4eHrwdRkRkotgJuoIK+/zY29vLXBKqKYW/W/bvIiIyXQxAlcTbXqaLv1siItPHAERERERmhwGIiIiIzA4DEJVq1qxZCAoKkrsYRERENYIByAwoFIpyt1mzZpV4zoQJExATE6P5ftiwYejbt6/eyuzv768pn52dHfz9/TFgwADs3r0bAHDv3j14eXlh7ty5JZ47YMAAdOjQAX5+fuXWe9iwYXqrDxERPSJJwOnTwKMpR+TCYfBm4M6dO5r9TZs2ITo6GgkJCZpjjo6Omn1JklBQUABHR0ed43L497//jZEjRyI3NxdXr17FN998g7CwMMyePRvTpk3DmjVr0L9/f/Tu3RuBgYEAgM2bN2Pbtm04fvw43NzcUFBQAAA4ePAg+vXrh4SEBDg7OwPQDnknIqIalpUF7N4N/Pab2K5fBxYvBqKiZCsSA1A1kCQgM1P/r2tvDzzJgCUvLy/NvouLCxQKheZYbGwsnn/+efz++++YPn06Tp06hZ07dyI2NhZbt25FfHw8Zs2ahfXr1wPQjpDas2cPnnvuOZw6dQrjxo1DXFwc7O3t0a9fP3z66aea8DRs2DCkpKSgU6dOWLx4MXJzc/H6669jyZIlsLKyKrfcTk5OmnLWq1cPnTt3hre3N6Kjo/Haa6+hT58+GDRoECIjI3Ho0CGkpKTg3Xffxfz589G0aVOda7m5uQEAPDw84Orq+vgfGhERVc3169rAs3u3CEGFbG2B+/flKxsYgKpFZiYgR2NJejrg4FA915oyZQoWLVqEBg0aoFatWoiNjdU8NmHCBJw7dw4qlQpr164FIAJFRkYGwsPDERoair///hvJycn417/+hTFjxmDdunWa5+/Zswfe3t7Ys2cPLl68iIiICAQFBWHkyJEVLue4ceMwe/Zs/Pzzz5g0aRKWLl2KwMBAzJ49G+fOnUPLli0xduzYqv44iIioovLzgb/+0oaeU6d0H/fzA3r1ElvXruKveBkxABEAcbvphRdeKPUxR0dH2NnZIScnR6c1af369cjOzsaGDRvg8CiJLV++HL1798aCBQvg6ekJAKhVqxaWL18OCwsLNGvWDL169UJMTEylApCbmxs8PDxw9epVAICzszPWrl2L7t27w8HBASdPnuQ8PkRE+vLgAbB9uwg8//sf8PCh9jGlEggN1YaewMAnu22hJwxA1cDeXrTGyPG61SU4OLjCzzl37hxat26tCT8A8Mwzz0CtViMhIUETgFq0aKGzpIS3tzdOPfrLYO7cuTodmc+ePYt69eqV+7qSJOmEnK5du6JDhw4ICgpC/fr1K1wPIiJ6QoUdmLdtE6EnLg5Qq7WP16oF9OghAk+PHoABLxnFAFQNFIrquxUlF4carEDxvj4KhQLqR/9hRo0ahQEDBmge8/HxKfda9+/fx927dxEQEKBz3NLSEpaW/OdMRFTtMjN1OzDfuKH7eMuWwEsvidDToQNgJO/FxlFKkp21tbVmRFWh5s2bY926dcjIyNAEqAMHDkCpVJbohFwWNzc3TQflJ7F06VIolUq9DsknIjI7167pdmDOztY+ZmsLdOsmAs+LLwJG2vLOAERPxN/fHzt27EBCQgJq164NFxcXDB48GDNnzkRkZCRmzZqFu3fvYuzYsRgyZIjm9ldVpKWlITExEXl5ebhy5Qq++eYb/Oc//8G8efPQqFGjaqgVEREBEB2Y4+K0oef0ad3H69XT9uV5/nnZOzBXBwYgeiIjR45EbGwsgoODkZ6erhkGv2PHDowbNw7t2rXTGQZfHaKjoxEdHQ1ra2t4eXmhQ4cOiImJwfPPP18t1yciMmtpacAvv4jAs317yQ7MHTtqQ0/LlgbVgbk6KCRJkuQuhKFRqVRwcXFBamqqZtK8QtnZ2bhy5QoCAgJga2srUwmpJvF3TEQm7dgxYPVqYONG3RE8bm5Az54i8ISHi++NTHmf38WxBYiIiMjUpacD330HrFkDHDmiPd6kCdCvn7YDc5ERu6aOAYiIiMhUxceL1p5vvxW3vADA2lqEnrffBjp3NrlbW0+KAYiIiMiUZGQA338vWnsOH9Yeb9wYeOstIDISqFNHvvIZCAYgIiIiU3DypGjt+eYbQKUSx6ysgFdeEa09zz9vtq09pWEAIiIiMlaZmcAPP4jg89df2uMNG4rWnmHDAA8P2YpnyJRyFwAAVqxYAX9/f9ja2iIkJASHizbZFfPFF1/g2WefRa1atVCrVi2EhYWVOH/YsGFQKBQ6W48ePWq6GkRERPpx+jTw3nuAjw8wfLgIP5aWwGuvAbt2AefPA5MmMfyUQ/YWoE2bNiEqKgqrVq1CSEgIlixZgvDwcCQkJMCjlF9cbGwsBg4ciI4dO8LW1hYLFixA9+7dcebMGfj6+mrO69Gjh2blcgCwsbHRS32IiIhqRFYWsHmzaO05eFB7PCAAGDlSBKEiC1ZT+WSfBygkJATt2rXD8uXLAQBqtRp+fn4YO3YspkyZ8tjnFxQUaFYbHzp0KADRApSSkoKtW7dWqkycB8i88XdMRAbl7FnRoXnDBu1khRYWQJ8+om/PCy+IiQupQvMAyfoTy83NxdGjRxEWFqY5plQqERYWhri4uCe6RmZmJvLy8kqsJxUbGwsPDw80bdoUo0ePxv3798u8Rk5ODlQqlc5GVaNQKCodQImIzF52tujM/OyzQIsWwNKlIvzUrw98/LFYkHTLFjFhIcNPpcj6U7t37x4KCgpKrBvl6emJxMTEJ7rG5MmT4ePjoxOievTogQ0bNiAmJgYLFizA3r170bNnzxKLeRaaN28eXFxcNJufn1/lK2WAiveHKr7NmjVL7iICEKG1sExKpRIuLi5o06YNJk2ahDt37gAAvv76azg4OODixYs6z719+7amJfBx9Y2NjZWhdkRET+Cff4CoKMDXFxgyBNi/X7T2vPwy8PvvwKVLwLRpgLe33CU1erL3AaqK+fPn4/vvv0dsbKzOrYrXX39dsx8YGIhWrVqhYcOGiI2NRbdu3UpcZ+rUqYiKitJ8r1KpTCoEFYYHQPS5io6ORkJCguaYo6Njha6Xl5cHKyuraitfcQkJCXB2doZKpcKxY8ewcOFCfPnll4iNjcWQIUPw008/YdiwYdi3bx+Uj/7yGTlyJNq2bYuRI0fitdde01xr3LhxUKlUOv3BKrL6PBFRjcvOBn76SfTt2btXe9zPT/TtefNNEYioWsnaAuTu7g4LCwskJSXpHE9KSoLXYzpyLVq0CPPnz8fOnTvRqlWrcs9t0KAB3N3dS7QaFLKxsYGzs7POZkq8vLw0m4uLCxQKheZ7Dw8PfPrpp6hbty5sbGwQFBSE7du3a5579epVKBQKbNq0CV26dIGtrS2+/fZbAMBXX32FFi1awMbGBt7e3hgzZozO6967dw+vvPIK7O3t0bhxY/zyyy9PVF4PDw94eXmhSZMmeP3113HgwAHUqVMHo0ePBgCsXr0a58+f1yy6um7dOhw4cABr166FjY2NTn3t7OxKHLO2tq6OHysRUeXl5AC//gq88YaYlHDQIBF+lEqgd29g2zbgyhVgxgyGnxoiawuQtbU12rZti5iYGPTt2xeA6AQdExNT4sO0qIULF2LOnDnYsWMHgoODH/s6N2/exP379+FdQ02GkiQhMy+zRq5dHnsreyiqOKnV0qVLsXjxYqxevRpt2rTBV199hT59+uDMmTNo3Lix5rwpU6Zg8eLFaNOmDWxtbbFy5UpERUVh/vz56NmzJ1JTU3HgwAGda3/00UdYuHAhPvnkEyxbtgyDBw/GtWvXKtwCY2dnh1GjRuH9999HcnIyPDw8sGbNGgwcOBCtW7fG+++/j6VLl5pUqx0RmaC8PCAmBti0SbT4pKZqH/PzEy09I0aIfapxst8Ci4qKQmRkJIKDg9G+fXssWbIEGRkZGD58OABg6NCh8PX1xbx58wAACxYsQHR0NDZu3Ah/f39NXyFHR0c4OjoiPT0dH330Efr16wcvLy9cunQJkyZNQqNGjRAeHl4jdcjMy4TjvIrdRqoO6VPT4WDtUKVrLFq0CJMnT9bcNlywYAH27NmDJUuWYMWKFZrzxo8fj1dffVXz/ccff4wPPvgA48aN0xxr166dzrWHDRuGgQMHAgDmzp2Lzz//HIcPH67UnEzNmjUDIFqkPDw80LdvXwwYMAA9evRA7969ERkZWeFrEhHVuPx8IDZWhJ4tW4AHD7SPeXsD/fsDERFiIVJ2ZtYr2QNQREQE7t69i+joaCQmJmpuwRR2jL5+/bqmnwcArFy5Erm5uTr9PABg5syZmDVrFiwsLHDy5EmsX78eKSkp8PHxQffu3TF79mzOBVSMSqXC7du38cwzz+gcf+aZZ3DixAmdY0Vb2pKTk3H79u1S+1MVVfTWpIODA5ydnZGcnAwAaNGiBa5duwYAePbZZ/G///2v3GsVztZQtMVrxowZ2LBhA6ZPn17uc4mI9KqgQHRe3rQJ+PFH4NH7HgAxMeFrr4nQ06kTQ4+MZA9AADBmzJgyb3kVH7Fz9erVcq9lZ2eHHTt2VFPJnoy9lT3Sp6br9TULX1dfHBy0LU12dnZP9JziHaUVCgXUajUA4Pfff0deXt4TX+/cuXMAAH9/f80xS0tLna9ERLJRq4G4OLEsxebNQJHBJ6hdG3j1VRF6unQRMzaT7PhbqAYKhaLKt6Lk4OzsDB8fHxw4cABdunTRHD9w4ADat29f5vOcnJzg7++PmJgYPP/885V67fr16z/xuVlZWVizZg06d+6MOlzBmIgMhSQBf/8tWno2bxZz8xRydRWLkEZEAF27ikVJyaAwAJm5iRMnYubMmWjYsCGCgoKwdu1axMfHa0Z6lWXWrFkYNWoUPDw80LNnT6SlpeHAgQMYO3ZslcuUnJyM7OxspKWl4ejRo1i4cCHu3buHLVu2VPnaRERVIklAfLwIPT/8IEZqFXJyEvP1REQA3bsDHHFq0BiAzNx7772H1NRUfPDBB0hOTsZTTz2FX375RWcEWGkiIyORnZ2Nzz77DBMmTIC7u3uJflmV1bRpUygUCjg6OqJBgwbo3r07oqKiHjs1AhFRjZAksfhoYei5cEH7mIODGLYeEQH06AFw+RyjIftaYIaIa4GZN/6OiQiAmJV50yaxPeqHCECEnJdeAgYMAHr1Auz11x+TyleRtcDYAkRERFTo4kVtS8/Jk9rj1tZAz56ipad3b6CCM+iT4WEAIiIi85aXJyYmXLZMDF8vZGUl+vJERIiV111c5CsjVTsGICIiMk/JycCaNcCqVcCtW+KYhQXQrZsIPa+8AtSqJW8ZqcYwABERkXk5ckS09nz/PZCbK455egJvvy02Hx95y0d6wQBERESmLzcX+O9/RfD56y/t8ZAQYOxYMTszVwswKwxARERkuhITgdWrxW2uR2tHwspK3OIaOxYoZ9JXMm0MQEREZFokCTh0SLT2bN4sOjkDYvHR0aOBt94St7zIrDEAERGRacjJEcPXP/9c9PMp1LGjaO159VXOzkwaDEBERGTcbt0St7jWrNGuvG5jAwwcKILP00/LWz4ySAxARERkfCQJOHhQtPZs2QLk54vjdesC77wD/OtfABdPpnIo5S4A6c+wYcOgUCigUChgZWWFgIAATJo0CdnZ2ZpzCh//q+goCQA5OTmoXbs2FAoFYmNjNcf37t2Lrl27ws3NDfb29mjcuDEiIyOR+2hoaWxsLBQKBVq0aIGCggKda7q6umLdunU1Vl8iMkFZWcDatUDbtkCnTuKWV34+0Lmz6O9z5QowdSrDDz0WA5CZ6dGjB+7cuYPLly/js88+w+rVqzFz5kydc/z8/LB27VqdYz/99BMci039fvbsWfTo0QPBwcHYt28fTp06hWXLlsHa2rpE2Ll8+TI2bNhQrXUpDFlEZAauXxfBxs8PePNN4PhxsSbXiBFidfa9e8VQdkve2KAnwwBkZmxsbODl5QU/Pz/07dsXYWFh2LVrl845kZGR+P7775GVlaU59tVXXyEyMlLnvJ07d8LLywsLFy5Ey5Yt0bBhQ/To0QNffPEF7OzsdM4dO3YsZs6ciZycnDLLdv36dbz88stwdHSEs7MzBgwYgKSkJM3js2bNQlBQEP7zn//oLFSqUCiwevVqvPTSS7C3t0fz5s0RFxeHixcv4rnnnoODgwM6duyIS5cuVfrnRkQykCRtsAkIAObPB+7fB+rVAxYsAG7eBP7zH6B1a7lLSkaIAag6SBKQn6H/TZKqVOzTp0/j4MGDsC42KqJt27bw9/fHjz/+CEAEk3379mHIkCE653l5eeHOnTvYt2/fY19r/PjxyM/Px7Jly0p9XK1W4+WXX8aDBw+wd+9e7Nq1C5cvX0ZERITOeRcvXsSPP/6ILVu2ID4+XnN89uzZGDp0KOLj49GsWTMMGjQIb7/9NqZOnYojR45AkiSMGTPmSX4sRCS3lBTgiy9EsHnuOeDHHwG1Gnj+edHf59IlYNIkoHZtuUtKRoxthdWhIBP4QYaVgQekA5YOFXrKtm3b4OjoiPz8fOTk5ECpVGL58uUlznvzzTfx1Vdf4Y033sC6devw4osvok6xe+r9+/fHjh070KVLF3h5eaFDhw7o1q0bhg4dCmdnZ51z7e3tMXPmTHz44YcYOXIkXIotKhgTE4NTp07hypUr8PPzAwBs2LABLVq0wN9//4127doBELe9NmzYUKIsw4cPx4ABAwAAkydPRmhoKGbMmIHw8HAAwLhx4zB8+PAK/ayISA+yssTtrL//Bg4fFl8vXNA+bm8PDBkCjBkDtGwpXznJ5LAFyMw8//zziI+Px6FDhxAZGYnhw4ejX79+Jc574403EBcXh8uXL2PdunV48803S5xjYWGBtWvX4ubNm1i4cCF8fX0xd+5ctGjRAnfu3Clx/ogRI1C7dm0sWLCgxGPnzp2Dn5+fJvwAwFNPPQVXV1ecO3dOc6x+/folwg8AtGrVSrPv+WiCs8DAQJ1j2dnZUKlUZf1oiKim5ecDJ06I1p233gLatAGcnIBnngHGjwc2btSGn2bNgEWLxG2uVasYfqjasQWoOljYi9YYOV63ghwcHNCoUSMAol9P69at8eWXX2LEiBE659WuXRsvvfQSRowYgezsbPTs2RNpaWmlXtPX1xdDhgzBkCFDMHv2bDRp0gSrVq3CRx99pHOepaUl5syZg2HDhlX6dpSDQ+ktXlZWVpp9hUJR5jG1Wl2p1yWiCpIk4OJF3Zad48dFi09xXl5Au3ZiWYp27YDgYN7eohrHAFQdFIoK34oyBEqlEh9++CGioqIwaNCgEh2X33zzTbz44ouYPHkyLCwsnuiatWrVgre3NzIyMkp9vH///vjkk09KhKPmzZvjxo0buHHjhqYV6OzZs0hJScFTTz1VidoRkV7dvq0NOoVbSkrJ81xcRMApGnh8fcX7KJEeMQCZuf79+2PixIlYsWIFJkyYoPNYjx49cPfu3RL9eQqtXr0a8fHxeOWVV9CwYUNkZ2djw4YNOHPmTJmdnQFg/vz5mr45hcLCwhAYGIjBgwdjyZIlyM/PxzvvvIMuXbogODi46hUlourz8KFYaqJo687t2yXPs7ERt7kKg067dkDjxoCSvS9IfgxAZs7S0hJjxozBwoULMXr0aJ3HFAoF3N3dy3xu+/btsX//fowaNQq3b9+Go6MjWrRoga1bt6JLly5lPq9r167o2rUrdu7cqfNaP//8M8aOHYvOnTtDqVSiR48e5QYpItKDzExtJ+XCwHPxYsnzlEqgRQtt2GnfXvTbKXIrmsiQKCSpimOpTZBKpYKLiwtSU1NLtH5kZ2fjypUrOvPQkGnh75jMniQBMTGiE/IffwDFJjYFADRsqG3Vad9etPSU0UePSF/K+/wuji1AREQk5OWJ5SQ++UTMrlyInZTJBDEAERGZu/R0MaPyZ5+JJScAMf/OiBFiNfVGjdhJmUwOAxARkblKTBSrqa9cqR2x5eEhQs/o0WzlIZPGAEREZG7++QdYvBjYsAEoXFS4SRPggw/ErMvFpsQgMkUMQERE5kCSgAMHRP+eX37RHg8NFetq9enD4elkVhiAiIhMWUGBCDwLFwJ//SWOKRQi8EycKJahIDJDDEBERKYoK0vc4lq8WLu+lo0NMHSouNXVtKm85SOSGQMQEZEpuX8f+L//A5YtA+7eFcdq1RKdmseOFUPaiYgBiIjIJFy5Anz6KfDVV2L2ZgCoXx94/30xnN3RUd7yERkY9ngjAGIpiq1bt8pdDCKqqCNHgIgIMVfP8uUi/LRpA2zcKJasGDeO4YeoFAxAZmTYsGHo27dvqY/duXMHPXv21G+BHvH394dCoYBCoYCdnR38/f0xYMAA7N69GwBw7949eHl5Ye7cuSWeO2DAAHTo0AF+fn6aa5S2DRs2TM+1IqpBkgT8739A165iZuYffgDUaiA8XCxdcfQoMHAgYMlGfqKy8H8HAQC89NAvIDc3F9bW1qU+9u9//xsjR45Ebm4url69im+++QZhYWGYPXs2pk2bhjVr1qB///7o3bs3AgMDAQCbN2/Gtm3bcPz4cbi5uaHg0XpFBw8eRL9+/ZCQkKBZC8aO85qQKcjNBb77TqzRdfq0OGZpCbz+OjBhAtC6tbzlIzIibAEiACVvgR08eBBBQUGwtbVFcHAwtm7dCoVCgfgi6wOdPn0aPXv2hKOjIzw9PTFkyBDcu3dP8/hzzz2HMWPGYPz48XB3d0d4eHiZr+/k5AQvLy/Uq1cPnTt3xpo1azBjxgxER0cjISEBffr0waBBgxAZGYm8vDzcvXsX7777LubPn4+mTZuiTp068PLygpeXF9zc3AAAHh4emmMuLi7V/jMj0huVSszf06ABMGyYCD+OjkBUFHD5MvD11ww/RBXEAFQdJAnIyND/Jkk1Uh2VSqVpaTl27Bhmz56NyZMn65yTkpKCrl27ok2bNjhy5Ai2b9+OpKQkDBgwQOe89evXw9raGgcOHMCqVasqVI5x48ZBkiT8/PPPAIClS5fi/v37mD17Nt555x20bNkSY8eOrVpliQxdXh7QubOYrPDWLcDbG5g/H7hxQwxx9/OTu4RERom3wKpDZqY8nQzT0wEHh2q/7MaNG6FQKPDFF1/A1tYWTz31FG7duoWRI0dqzlm+fDnatGmj0y/nq6++gp+fH86fP48mTZoAABo3boyFCxdWqhxubm7w8PDA1atXAQDOzs5Yu3YtunfvDgcHB5w8eRIKLtBIpm7dOuDECcDNTdz6GjRIzOdDRFXCFiAqISEhAa1atYKtra3mWPv27XXOOXHiBPbs2QNHR0fN1qxZMwDApUuXNOe1bdtWsz937lyd868XrjpdDkmSdEJO165d0aFDBwwZMgT169evdB2JjEJWFvDRR2J/xgxg+HCGH6Jqwhag6mBvL1pj5HhdmaSnp6N3795YsGBBice8vb01+w5FWqhGjRqlc4vMx8en3Ne4f/8+7t69i4CAAJ3jlpaWsOToFjIHK1eK215+fsCoUXKXhsik8FOkOigUNXIrSi5NmzbFN998g5ycHNg8+mvz77//1jnn6aefxo8//gh/f/8nDiNubm6aDspPYunSpVAqlWUO3ScyaSoVUHiLeeZMoEiLLBFVHW+BmZnU1FTEx8frbDdu3NA5Z9CgQVCr1Xjrrbdw7tw57NixA4sWLQIAze2od999Fw8ePMDAgQPx999/49KlS9ixYweGDx+uGY5eEWlpaUhMTMSNGzewb98+vPXWW/j4448xZ84cNGrUqOoVJzI2n30mlrVo0gSIjJS7NEQmhy1AZiY2NhZt2rTROTZixAid752dnfHrr79i9OjRCAoKQmBgIKKjozFo0CBNvyAfHx8cOHAAkydPRvfu3ZGTk4P69eujR48eUCornqujo6MRHR0Na2treHl5oUOHDoiJicHzzz9f+coSGat798QILwCYPZsTGhLVAIUk1dBYaiOmUqng4uKC1NRUzUR6hbKzs3HlyhUEBATodBI2dd9++y2GDx+O1NRUk59U0Fx/x2RAJkwQAahNG7HURSX+qCAyR+V9fhfHPyuoVBs2bECDBg3g6+uLEydOYPLkyRgwYIDJhx8i2d28Kdb0AoA5cxh+iGoIAxCVKjExEdHR0UhMTIS3tzf69++POXPmyF0sItM3ezaQkwM8+yzQo4fcpSEyWQxAVKpJkyZh0qRJcheDyLxcuAB8+aXYnztXjDAlohrBtlUiIkMxcyZQUAC8+CLQqZPcpSEyaQxAlcS+46aLv1uSxYkTYqV3QPT9IaIaxQBUQVZWVgCAzMxMmUtCNaXwd1v4uybSi2nTxNeICCAoSNaiEJkD9gGqIAsLC7i6uiI5ORkAYG9vzwU5TYQkScjMzERycjJcXV1hYWEhd5HIXBw4APz2G2BhAfz733KXhsgsGEQAWrFiBT755BMkJiaidevWWLZsWYnFNwt98cUX2LBhA06fPg1ALLY5d+5cnfMlScLMmTPxxRdfICUlBc888wxWrlyJxo0bV0t5vby8AEATgsi0uLq6an7HRDVOkoAPPxT7w4eLmZ+JqMbJHoA2bdqEqKgorFq1CiEhIViyZAnCw8ORkJAADw+PEufHxsZi4MCB6NixI2xtbbFgwQJ0794dZ86cga+vLwBg4cKF+Pzzz7F+/XoEBARgxowZCA8Px9mzZ6tlYjuFQgFvb294eHggLy+vytcjw2FlZcWWH9KvnTuBffvEKu/R0XKXhshsyD4TdEhICNq1a4fljyb+UqvV8PPzw9ixYzFlypTHPr+goAC1atXC8uXLMXToUEiSBB8fH3zwwQeYMGECALH+laenJ9atW4fXX3+9xDVycnKQk5Oj+V6lUsHPz++JZpIkIqo0tRpo1w44dgx4/33g00/lLhGRUavITNCydoLOzc3F0aNHERYWpjmmVCoRFhaGuLi4J7pGZmYm8vLyNKuMX7lyBYmJiTrXdHFxQUhISJnXnDdvHlxcXDSbn59fFWpFRPSEtmwR4cfREZg6Ve7SEJkVWQPQvXv3UFBQAE9PT53jnp6eSExMfKJrTJ48GT4+PprAU/i8ilxz6tSpSE1N1WzFV0cnIqp2+fnA9OliPyoKqFNH3vIQmRnZ+wBVxfz58/H9998jNja2Sn17bGxsYGNjU40lIyJ6jK+/BhISADc34IMP5C4NkdmRtQXI3d0dFhYWSEpK0jmelJT02FE4ixYtwvz587Fz5060atVKc7zweZW5JhGRXuTkALNmif2pUwH2NSTSO1kDkLW1Ndq2bYuYmBjNMbVajZiYGISGhpb5vIULF2L27NnYvn07goODdR4LCAiAl5eXzjVVKhUOHTpU7jWJiPRm9Wrg+nXA1xd49125S0NklmS/BRYVFYXIyEgEBwejffv2WLJkCTIyMjB8+HAAwNChQ+Hr64t58+YBABYsWIDo6Ghs3LgR/v7+mn49jo6OcHR0hEKhwPjx4/Hxxx+jcePGmmHwPj4+6Nu3r1zVJCIS0tOBjz8W+9HRgJ2dvOUhMlOyB6CIiAjcvXsX0dHRSExMRFBQELZv367pxHz9+nUoldqGqpUrVyI3NxevvfaaznVmzpyJWY+alCdNmoSMjAy89dZbSElJQadOnbB9+/ZqmQOIiKhKli4F7t4FGjUSEx8SkSxknwfIEFVkHgEioif24AHQoAGQmgps3AgMHCh3iYhMSkU+v2VvASKiKkpKAr78EkhLA/z8gHr1xFc/P6BWLYBr1RmOhQtF+GnVSix6SkSyYQAiMlZXrgCffAJ89ZUYVVQaBwdtGCoajIru29vrt9zm6s4d4PPPxf6cOYBS1jEoRGaPAYjI2Jw6BSxYAHz/PVBQII6FhADt2wM3bojRRTduiH4mGRnAP/+IrSy1a5cfkHx8ACsr/dTNlM2eDWRlAaGhQK9ecpeGyOwxABEZiwMHgPnzgW3btMfCw8U8Mp07l7zVlZUF3LypG4qK7l+/LkYk3b8vtvj40l9XqQS8vcsOSYGBYiFPKtvly8AXX4j9efN4W5LIADAAERkySQK2bxcfmn/+KY4pFMBrrwFTpgBPP132c+3sgMaNxVbWtVNTyw9IN28CeXnArVti++uvktdp0gTYv59LOZRn5kyx9EX37kCXLnKXhkg2WVnAmTPAyZPi7SsoSL6yMAARGaL8fOC//xUtPidOiGNWVkBkJDBpUtmhpiIUCsDVVWyBgaWfo1YDycllB6R//gHOnxcdenfuBCz5llLC6dPAt9+K/blz5S0LkZ5Ikuj2duKE7nb+vPbO/YwZDEBEVCg7G1i/XnRuvnRJHHNwAEaNAt5/X8wcrE9KJeDlJbb27Us+fuYM0KEDsGcPMHEi8Nln+i2fMZg+XXwa9OsHtG0rd2mIql1uLnDuXMmwc+9e6ee7uwOtWwMBAfotZ3GcB6gUnAeI9E6lAlatEgHi0ezmqF0bGDdOLJXg5iZv+crz00/Aq6+K/fXrgaFD5S2PITl0SAREpVK0BDVvLneJiKokOVk35Jw8KcJPXl7Jc5VKoGlTEXaKbt7eNdcNjvMAERmL5GQxNHrFCiAlRRzz8wMmTABGjBCtP4bulVdEW/bs2cBbbwFPPQUUW6PPbH34ofg6dCjDDxmV/HwgIaFkq07h32fFubrqhpxWrYAWLQx7pRe2AJWCLUBU465dAxYtEhMYZmWJY82bA5Mni9mBra3lLV9FqdXAyy+LEWp16wJHjgCPlrMxWzExQFiY+F2ePw/Ury93iYhK9eCBaMkpGnTOnCl9ejGFQqziUrxVx8/PMAY3sgWIyFCdOSPm8Nm4UdsTsH17MZS9Tx/jnRxPqQS++UbMR5SQAPTvLwKAuc4fJEna1p9Roxh+SHb5+cDVqyKLX7ig/XrunBjsWRpHR9GSUzToBAYaR8P0k2AAItKHv/4SQ9l/+UV7LCxMBJ/nnzeMP52qysUF+PlnEej+/FN02l6+XO5SyePnn4HDh8UnRWEQIqpharWYraJowDl/XmyXL4sQVJaAgJKtOv7+xvs32ZNgACKqKZIkhobPnw/ExopjCoXoMDxlimn2k2naVLQE9ekj+jW1aSP6MpmTggJg2jSxP348bwVStZIkMbqqMNgUDTsXLmjvqJfGzk7cvmrSRGyNG4uvLVuKv1/MDQMQUXUrKAC2bBHB59gxcczKChgyRMzh07SpvOWrab17A//+NxAdDbzzjugJ2aGD3KXSn40bgbNnxUK0EybIXRoyUqmp2lBTPOykppb9PEtLoEED3YBTuO/ra9otOhXFAESm6Y8/xCxchX38i34t7Vh1PZaRIRYnvXBBfO/gIEZGRUWJzsHmYto04PhxMUS+Xz/RKdrbW+5S1bzcXDHrMyA6tLu6ylocMnzZ2UBcnLhjWjToJCWV/RyFQnQ6Lt6S06SJ6G5mrl3vKooBiEzPDz+ImYnl5OYGvPceMGaMmM/H3CiVYk6ghATRGtKvn5gs0dTXDPviC+DKFTFx5NixcpeGDFB2tpgeas8ecWf8r79KH20FiLunpbXkNGxo2MPLjQUDEJmW7GxxmwkQC80Urk+lUGg7Gpf29UmPPe4xpVLc7hkxQgyhMGdOTsDWrUC7duJP3PfeA1avlrtUNScjQ8yFBIh5kezt5S0PGYScHNG6Uxh44uLE21RRXl5iPeOnntINOpyFpWYxAJFpWbJEzLFTt64YicQPIXk1bgx89x3QqxewZo0IpW+/LXepasayZeK+RUAA8K9/yV0akklurgg8sbEi9Bw8WDLweHqKwZ/PPSe2Jk1MYyCoseFEiKXgRIhGKilJfOCmpQFffw288YbcJaJC8+eLIf9WVuJT4Zln5C5R9UpJEcEnJQXYsEF0eCezkJcH/P23NvAcOFByJJaHhzbsPP+8GAfBwFMzOBEimacZM0T4adcOGDRI7tJQUZMnixFxmzeL/kBHj+p/Ydea9MknIvy0aMF/eyYuL0/88y28pbV/P5CZqXuOu7s27Dz3nJjknYHH8DAAkWk4eVIsKwEAn37KsZ7VJCdHrNOalia+Ft/S0sSP2sWl7M3ODlAoFMDatcA//wCnTom5kPbuBWxt5a5i1SUliVuvAPDxx4CFhazFoeqVny+ye9HAk56ue07t2toWnueeEzmYgcfwMQCR8ZMk4IMPxDSo/fsDnTrJXSJZqdXiDbp4UCkrwJR3LDe36uWxsioMQw54ynYrvrEMhvPhw9gX+A5+7PklXFwVcHERI8bLClG2tgb8gTJnjmgCaN9erIdGRi0/H4iP1waeP/8U/yeKcnMDunTRtvK0aMG/uYwR+wCVgn2AjMy2bWLyPWtr0cIQECBbUW7eFFtenggPhV+L7lf02JOen5EhQkvxv06rg4ODGNTl7Ky7OTmJeR9TU0tuKpV2eqSiwrAL29EDFlDjXSzH/+Hdx76+NkRpg1KHDsD06TIPB752TfQ7y8sTc0916yZjYaiovDzx7zAlRWxF98vaUlPFr1Sl0r2Wq6sIPIW3tAIDGXgMFfsAkfnIy9POtjt+vKzh58AB8QaZlydbEXRYWpYMLIWhpSLHHB3FtSqqsCWqaCgSHzIv4NCPC9FxywR8rhyPRr1b4ohDl3JDVF6emP7/3j3t9XfvFtn3hx9knFx71ixRuG7dGH6qWW5uxYJL8WMZGZV/bRcX3RaewEDe2TRFbAEqBVuAjMiyZWJ+mTp1xPSpMi1ok5YmFg+8ckWM+KhVS7RaWFtrvxbdf5LHKnK+lZVopSkaYGxsDPi2kSSJUXobN4rf3ZEjQL16JU5Tq8XPtngwunlT9Hm/e1fUe/VqYPBgPdfh3DmxiJJaLWazCwnRcwFMQ0EBcPEicOKE7lbWCuUV5eQkWnAKt8JWxLI2Dw8xHw8Dj3FiCxCZh4cPxV/ggFh7SsbV/N5/X4Sf+vXFm7c5LixYIQqFmDX53DmxZMYrr4jepcXuZxXtYF1c794i9MTGiiy1Zw/w+ed6nPppxgwRfvr2Zfh5QiqVGK9QNOicOlX+Ap7OzuUHlvKCjbNz5VovyTywBagUbAEyElFRwGefiR6I8fGyvdP9/LP4DFQoxIdx586yFMM4XbsGBAeLe1tvvCHm0KlAs1VBgci+s2eLRqUWLcQtsaeeqsEyA6LFql07UdaTJ0VLEGmo1eIPgqJB5+RJcaw0dnbiNlPr1tqteXMRYtgSQxVRkc9vBqBSMAAZgQsXxKddXh6wYwfQvbssxUhKEm/cd++KFTgWLJClGMZtzx7ghRdEmvnsM9GXq4J27xbT7yQliRagFSuAYcOqvaRa4eHAzp1iwsMNG2rwhQxfRoZoxSneqlN85FShunV1g07r1kCjRgw6VD0YgKqIAcgIvPKKWGeqZ0/g999lKYIkiVHPv/4KtGolpr839bU+a8znnwPjxolPwZ07ga5dK3yJpCTRiPTHH+L7oUNFEKr2JdliY0XPWEtLsdhrgwbV/AKGSZKAGzdK9tW5eLH00X7W1uJvlOJhx81N/2Un88EAVEUMQAZuzx7xAWlhIf7UbN5clmL85z/AyJHijf7IEdESRJUkScDw4WIF+dq1xQ/U37/ClykoAObNA2bOFLdhmjUTt8Sq7XcjSWIZj7g44J13RMIyUcnJomUtLk57C+vhw9LP9fQsGXSaNhWd84n0iQGoihiADFhBgegzEh8PvPsusHy5LMW4dEm8yWdkAIsWiXkYqYqys4FnnxXhp3VrsYpkJXs0790rbondvi0mUVy2DBgxohpGxf36K9Cnj+i0cukS4O1dxQsajsxMMenfH38Au3aJ0FOcpaX4e6N1a9HqWRh2PD31X16i0jAAVREDkAFbuxZ4800x5OPiRbHojp7l54uOznFxYp6QmBhOilZtbtwQATc5GXj9dTFMvpKp5e5dcRts+3bx/aBBwKpVYlh0hZ05A2zaJEauJSaKtc3mz69UuQxFQYFY0+qPP8R24EDJmb9btxbz4bRpAwQFifDD27xkyBiAqogByEClp4tZdxMTZW12mTsXmDZNDLE9eVIMfadq9Oef4hZnfj6wcCEwcWKlL6VWi3VKp00TH/iNG4tbYkFBT/DkCxdE6Pn+exGACtWvLxaHMrLOLJIkGq127RKBZ/duMWFgUX5+oj96WJj4FbBlh4wNA1AVMQAZqOhoMd65YUPxgSTDn6LHjokpX/LzxeCfIUP0XgTzsHKl6GOjVAL/+1+VR/kdOCAalG7eFP9sPvsMGDWqlMalK1dEQtq0ScxPVMjKCujRA4iIELfAKtWMpH9374qgU3hb69o13cddXER/7sLQ07ixAU+eSfQEGICqiAHIAN24IXpVZmUBP/4oVhPXs6wsoG1bMXffa6+Jz0l+WNQQSQLeekv0NK9VC/j7bxF8q+D+fTE0fts28X3//uKOlkvaTW3oOXxY+wQLC5EMIiLERE+urlV6fX3IytL24/njD90MB4gc17GjCDsvvCD+PXOiQDIlnAmaTM+HH4p3986dxRB4GUydKsKPt7foS8LwU4MUCtHB/fRpscxE376i01UVxrTXrg388oto/fl0UiI8Nv8X53/dhHbZ+7UnKZWiY1dEhAjZMvQxq4iCAhFyCm9rHTgA5OTontOqlQg8YWHiv4+DgzxlJTI0bAEqBVuADMzhw+K+k0IhWgLattV7Ef74Q/zFDIg7Mj166L0I5un2bfH7TkyserPbvXui9XDTJkh790KhVmseutXwWfiMj4DitX6Al1c1Fb76SRJw+bL2ltbu3SWHptetq9uPx4CrQ1Tt2AJEpkOSxJIXgBjSI0P4efhQO6vwO+8w/OiVjw+wZYsYivTf/4qRV1OnPvnzHz4UE2Zu2iRSQ0EBAEABIL9tCDbkRCD6dH/culQXr+wGvhwM1KqRilTezZvittaePSL0XL2q+7izs24/niZN2DpJ9CTYAlQKtgAZkM2bgQEDxHww588Dvr56L8KgQcB334kPlmPHeAtBFoWzTioUohPPiy+Wfa5KJe51bdoklknJy9M+9vTT4vbWgAGAvz8kScwRNGGCOM3fXwz6kmttU0kSk0v/+ad2Kx54rKyA0FBtP57gYPbjISrETtBVxABkILKzxcQjV6+KVd9nztR7Eb77TgQgCwsxL1/79novAhV65x0xOszFRdwWbdJE+1hGhghGmzaJpVGKdoRp2VKEnogIMcypFEeOiIcvXxZhYsEC4P33a74lJT9fzOlZGHb27xcjt4pSKsU8PJ07a/vxVPvyHkQmggGoihiADMTChWLCOV9f8WexnptebtwQHUhTUmTLX1RUbi7QrZtICc2bi+me9+8XoefXX8VUxoWaNtWGnidcGj41VTQybd4svu/dG1i3rnqn+8nKAg4d0gaeuDgxvVVRtraiBerZZ8UWGmo0o+6JZMcAVEUMQAYgOVksEZ2WJtaHGjpUry+vVoupZ2JiRKvP/v1c18ggJCaKez63bonmmaJvXwEBYrKfiAiRXCvRfCNJYoTf+++LRiQ/P3FLrGPHyhX34UMxMqsw8Bw5ontHDhCj6595Rht42rblbMtElcVO0GT8oqNF+GnbVizxrWfLlonwY2cHfP01w4/B8PICfvpJJIXChDJggAg9wcFVvmelUACjR4tWlwEDxGTQnTsDc+aICakft+TJrVu6/XdOny65UrqPjzbsPPusuEPHpVSI9I8tQKVgC5DMTp8WixCp1cC+feJTQo/OnhV9ZXNygP/7P/GBSAYmIUHcswoOrrH0kJYmZoveuFF837OnaIysU0d8L0miX37RwHPlSsnrNGmiG3gCAjhKi6imsAWIjJckiTW+1GqgXz+9h5/cXNHglJMjPvBGjdLry9OTatq0xl/CyQn45hsxxHzsWDH/U1AQ8O67YhHR/fvFndqilEpxTmHY6dSJ62kRGSq2AJWCLUAy+v13oFcvwNpaNMVUcfmDipo2TSx2Wrs2cOqUmPWZ6NQpcUvsn390j9vYlOywzLcMIvmwBYiMU16edoX3997Te/g5cEDMswcAa9Yw/JBWYKDowPzRR+K2V4cOIvAEB7PDMpGxYgAiw7FmjfgT291dNMXoUVqaGGimVgORkbKstUoGzsFBzMxARKaBYw/IMKSkaCfa+egjva+8HRUlJsGrXx9YulSvL01ERDJgACLD8PHHwP37YtK6t97S60v/8otYaUGhEKN8XFz0+vJERCQDBiCS38WLwOefi/3Fi/W6sFFyMvCvf4n9CRPEmptERGT6GIBIfpMniw7Q4eF6XWpdksTSB3fvik6us2fr7aWJiEhmsgegFStWwN/fH7a2tggJCcHhw4fLPPfMmTPo168f/P39oVAosGTJkhLnzJo1CwqFQmdr1qxZDdaAqmTvXmDLFjGByuLFen3pr74St7+srcV8LxzNQ0RkPmQNQJs2bUJUVBRmzpyJY8eOoXXr1ggPD0dy8dnFHsnMzESDBg0wf/58eHl5lXndFi1a4M6dO5pt//79NVUFqgq1WvQ+BkS/nxYt9PbSly4B48aJ/TlzxNJRRERUgyQJyE4G7h0Grm0CUk7JWhxZh8F/+umnGDlyJIYPHw4AWLVqFX777Td89dVXmDJlSonz27Vrh3bt2gFAqY8XsrS0LDcgkYH4+mvg2DExc9y//623ly0oEEPeMzJEn5/339fbSxMRmbbcFCDjKpB+RWwZV4D0q+JrxlUgP0N7bstowDVQnnJCxgCUm5uLo0ePYurUqZpjSqUSYWFhiIuLq9K1L1y4AB8fH9ja2iI0NBTz5s1DvXr1yjw/JycHOTk5mu9VKlWVXp+eQEYG8OGHYn/6dO0CS3qwcCFw8KBY6mD9esDCQm8vTURk3PIzgIxrugGnaODJS3nMBRSAnQ/gGCC+yki2AHTv3j0UFBTAs9hCOZ6envin+HzzFRASEoJ169ahadOmuHPnDj766CM8++yzOH36NJycnEp9zrx58/DRRx9V+jWpEj75BLh9W6wM+d57envZ48fFQvMAsHy5mPeHiIgeKcgVASfj6qPWmyIBJ+OKuIX1ODZ1RMBx8H/0NUD7vUN9wMIwOlya3EzQPXv21Oy3atUKISEhqF+/Pn744QeMGDGi1OdMnToVUYV9USBagPz8/Gq8rGbr5k3tlLoLF+qt93FWlljoND9frLM6ZIheXpaIyHBIaiDrNpB++VGrzWVtyMm4AmTeAvCYJUKtXIoEmkfhRvO9P2DlWOPVqA6yBSB3d3dYWFggKSlJ53hSUlK19t9xdXVFkyZNcPHixTLPsbGxgQ2HAOnPtGkijXTqJJKInnz4oVhf1csLWLVKTHxIRGRyclMftd5cLhZ0Lougo84t//kWdtqWm8JWHE1Ljj9gXUsPlah5sgUga2trtG3bFjExMejbty8AQK1WIyYmBmPGjKm210lPT8elS5cwhH/uG4YjR4ANG8T+p5/qLYX88QdQOGvCV1+J5caIiIySOg/IuF4k1FzRDTq5D8p/vsIScKgHODYocnuqSCuOrYdZ/IUo6y2wqKgoREZGIjg4GO3bt8eSJUuQkZGhGRU2dOhQ+Pr6Yt68eQBEx+mzZ89q9m/duoX4+Hg4OjqiUaNGAIAJEyagd+/eqF+/Pm7fvo2ZM2fCwsICAwcOlKeSpCVJ2iFXQ4YAj0b01bSHD4Fhw8T+6NFAkbukRESGR5KAnLultN482s+8IW5llcemjgg4jgGPvhaGnQaAfV1AaXI9YCpM1p9AREQE7t69i+joaCQmJiIoKAjbt2/XdIy+fv06lErtVEW3b99GmzZtNN8vWrQIixYtQpcuXRAbGwsAuHnzJgYOHIj79++jTp066NSpE/766y/U0eMoIyrDli3A/v2AnR0wd67eXvbdd4Fbt4DGjUXfayIi2UkSkJ0EqM4Bqn8A1XndoFN0uHhpLGx1Q03xoGMk/XDkpJAk6TG9ncyPSqWCi4sLUlNT4ezsLHdxTENOjljo9PJlMQxLT6PuvvsOGDRIDHU/cAAICdHLyxIRCep8IP3So5DzD5B6Trufl1rOExWAvW/J1pvCsGPrZRa3qSqqIp/fbAMj/Vi2TIQfb29g0iS9vOTNm8A774j96dMZfoioBuWlaYNN0aCTflH02SmNQimCjXNzwLkp4NRQG3QMaLi4qWIA0qPTp4GtW4Fu3YDgYMDKSu4S6cndu9qVRufOBRwcavwl1WrR7yclRXQ1mjatxl+SiEydJAFZd4rctioSdLJulf08C3vAuZl2c2kmQo9TI3Eri2TBAKRHv/wCzJghNkdHoHNnoGtXEYhatRLrgZqkmTMBlQp4+mmxBoUeLF8OxMSI7kbffGNGYZOIqk6dB6RdLP22VX5a2c+z9SoWdJqLr/Z1RWsPGRQGID1q1Qp47TVgzx7g/n3g99/FBgBubsDzz4sw1LUr0KSJidzePXMGWL1a7H/6abWmPLVa3Oa6eFG7Xbokvp45I85ZvFj8LImISpWdDDw4Djw8Djw8BqScBNIuAVJ+6ecrLADHhiVDjnNTk5kfx1ywE3QparoTtFoNnDwJ7N4ttr17gfR03XN8fUUQKtzKWcrM8EiSmOgwLQ2IjAR27ABeeUWMAqugvDzg2rWSAefiRdGlKLec+bwGDAC+/95EgiQRVY0kAZk3Rch5cEwEngfHyr51ZelYSshpBjg2Aiys9Vt2emIV+fxmACpFjQWgr78W92bs7cW9mUdfC2ztkZRmjyt37JBwwx4J1+2gKrBHFuyQCfHV1dseLdvbIyjUDsGd7VG7rvb5sLWtWsuKWi0WJ01PF6ElLa3q++oic1RYWYkpmB/N1VRcdrYIM6WFnGvXxOrtZbGyAho0ABo2FJdv1Ei737gxww+RWZLUohWnaNh5eAzIuV/KyQrAqTHg9jRQ62mgVmvA5SnAzpdvIEaIo8AM1fXrwOHDJQ5bAPB5tD1T1nPvAPj50VYaOzudUFXiq62tSBqlhZWMDPHXUU1wcgKmTUOaZyNcii8ZcC5eFHP0lPfydna6wabovp8fV3MnMmvqPNE350HRsBNfel8dhaUIN5qw00YEHqvSF8om08YWoFLUWAvQxYvAuXNAZqa4RZSZqbtf/Ouj/YK0TGTcz0JuSibUGVmwzs+EPTJhjTKGVlaWUil6Zzs5ab+Wsq92dEKOlRPS4Yg0OCEl3xEP851wP9cJ97IdkZTphDvpTrijcsDd+0pcvQoUW/KtBGdnbbgpHnK8vfmHGBEBKMgGUk49CjrHRN+dlJOAOqfkuRa2gGsrEXTcHoUd15YcdWXi2AJkqAo/2SvIAkDRX+O9e8DPe4DYP/IRtzsLNy9mwR6ZsIP46mKZiaebZ6Fdi0wENc1CQ+9MWOZni1agIoFG7eCENKlIeMmww/0HCjx4AM12//6j/cvQOV70DteTqlNHtxWnaMipXZshh4iKyFOJlpwHj25fPTwOpJ4FpFLuiVs5A7WCdMOOczMu90DlYgtQKYxtJugbN8TIst27xdDvmzd1H3dwADp2BCwtdUPMw4eVCzJFr+vmpt1q1y79+3r1RNBxcalaPYnIhKjzxJpW6VfECuXpV8USEIX7ZXVOtnHXDTpuT4uJAznMnKCHTtDr16+Hu7s7evXqBQCYNGkS1qxZg6eeegrfffcd6tevX7mSGwhjC0BFSZK401Y4wmz3btFiVB4Hh7LDS1nf16olGpSIiEqlzhejrjKuPgo1V3T3s249fkFPez9tyCn8ys7JVI4aD0BNmzbFypUr0bVrV8TFxSEsLAyfffYZtm3bBktLS2ypxHBnQ2LMAag4tVrMQB0XB1hblww0tWoBNpxtnYgqSl0AZN1+tEL51Ufhpsh+5o3Sb1cVZWELOPhrN8eAIvsNAFv3Gq0CmZ4a7wN048YNNHrUl2Xr1q3o168f3nrrLTzzzDN47rnnKnNJqiFKpZiAsVUruUtCREYn5/6j9ayKtt5cfXSr6nrZkwUWUlqLNa0cAgBH/0fhpsi+rSdbc0g2lQpAjo6OuH//PurVq4edO3ciKioKAGBra4usrKxqLSAREdUwSQ2kXwYenhAdjx/GAynx4hZWeZRWgH29Ulpv/EXQsfNi3xwyWJUKQC+88AL+9a9/oU2bNjh//jxefPFFAMCZM2eMvv8PEZFJy88CUs/oBp2HJ4D89NLPd6j/aHVy/5JBx84HUHIiLjJOlQpAK1aswPTp03Hjxg38+OOPqF27NgDg6NGjGDRoULUWkIiIKin7rjboPIwHUk6IW1ql9c1R2gCugY+GkwcBrq2BWq3EEHMiE1TpYfDZ2dk4efIkkpOToS42lrpPnz7VUji5mFInaCIyA5JarF5eGHIKA0/W7dLPt3EvEnQefXVuynlzyOjVeCfo7du3Y+jQobh//z6K5yeFQoGC8hZvIiKiysvPBFJOP7p19WhLOQnkZ5RysgJwalQy7NhxenWiSgWgsWPHon///oiOjoanp2d1l4mIyLxJEpCdLDomF26qsyLspJ0vff4czdIPQdpbWK6BXOeKqAyVCkBJSUmIiopi+CEiqqyCbDGkPP2SbtAp3Aoyy36urYe2Nadwc2rMW1hEFVCp/y2vvfYaYmNj0bBhw+ouDxGRaZAkIDup9HCTfrnspR40FGImZMcGYnNqrA07dl56qACRaatUJ+jMzEz0798fderUQWBgIKysrHQef++996qtgHJgJ2gieiL5WY9mPy4j5BQ8Zl40S6dH4aahNug4FH6tB1hwmnaiiqjxTtDfffcddu7cCVtbW8TGxkJRpDOdQqEw+gBERFSqmz8DN7YUacUpY5RVIYVStxWnaMBxbADY1GZnZCKZVCoATZs2DR999BGmTJkCpZKzfBKRGbi2CTjwesnjVs6AY0PdkFO42dcDLKz1X1YieqxKBaDc3FxEREQw/BCRebgbB8RFiv36g4C6fbShx7oWW3GIjFClEkxkZCQ2bdpU3WUhIjI86ZeBfS8D6hzAtw8QugGoHwHUDgZs3Bh+iIxUpVqACgoKsHDhQuzYsQOtWrUq0Qn6008/rZbCERHJKjcFiO0F5NwFarUBOn7Lta+ITESlAtCpU6fQpk0bAMDp06d1HlPwryEiMgXqPODP18TaWXa+QJdfAStHuUtFRNWkUgFoz5491V0OIiLDIUnA36OBpBjA0gF4bhtg7yt3qYioGrEXMxFRcecWApe+FMPYn9kkJh8kIpPCAEREVNT1/wLxU8T+00sA316yFoeIagYDEBFRoXuHgLghYr/JWKDpWHnLQ0Q1hgGIiAgQC5Pu6yMWKfXpBTz9mdwlIqIaxABERJSbCuztBWQnA66tgWe+43B3IhPHAERE5k2dB+zvD6SeBex8xIgvKye5S0VENYwBiIjMlyQBR8YAibsAC3sx1499XblLRUR6wABERObrn8XAxTUAFOK2l9vTcpeIiPSEAYiIzNONn4Djk8T+05+KBU6JyGwwABGR+bn/N3BwMAAJaPwO0HSc3CUiIj1jACIi85JxHdjbByjIArx7Am2XckV3IjPEAERE5iNPJVZ3z04EXAOBTt8DykotiUhERo7/84noyajzgPQrQNp5QHVefE27CDg1BFpMAxzqyV3C8qnzgf0RQOppwNYL6LINsHKWu1REJBMGICLSkiQg67Y25KgSHgWd80D6ZUAqKPmcpBjg8nqg6XtAi6mAdS39l/txJAk4+h5wZztgYSeGuxt6YCOiGsUARFRd8lTA3QOipcS6lu5mYWdY/UxyH2pbcTStOY/2CzLLfp6FHeDUBHBuIr46+ANXvwWSY4FznwCX/iNag5q8C1jY6qs2j5ewFLiwEoAC6LgRqB0sd4mISGYKSZIkuQthaFQqFVxcXJCamgpnZzaRUxkKcoB7fwGJf4hWkPuHS28hAQClTclQZO1W8phNKccqGyTys4D0iyUDTtp5IOde2c9TWACODUTAKRp2nJuImZIVxboOShJw+39A/GRxewkA7OsBrecA/oNKnq9vN38B9vUFIAFtFgHNP5C3PERUYyry+c0AVAoGICqVpAYentAGnuR9YiRRUY4NAZvaooWlcCsrFD0pC7snC08593WDTub18q9r56sNN0WDjmMAoLSqeDnVBcCVDcDJGUDWLXGsVhAQtBDwfqHi16sOD44Bu54VrVqN3gbarTSsljgiqlYMQFXEAEQARMtG+mURdhL/AJJ2i5BRlK0H4NkN8AoDvLoBDvVLXiM/TTcQ5TzQ/T73IZBbyrG8FBG6qsLKFXBuWrIlx7ERYOVYtWuXJT9L3HI6O0/cFgQArxeAoAWAW5uaec3SZN4EdoSIPk1eLwDP/Va5YEdERoMBqIoYgMxYdjKQuBtI+gNIjAEyruo+bukIeHTRBh6XljXXoiCpRYB4krCU80CMaNIJO01Fa5RcLR4594HTc4ALy0W/KADwfwNo/XHJoFjd8tJEy0/KCcClBfDCAcDapWZfk4hkxwBURQxAZiQvXdzKKmzlSTmp+7jCEnAPFWHHKwyo3Z6tCBWVfgU4MR24tlF8r7QGmowFWnwo+jxVN3W+6PNz+zfRQtf9EODoX/2vQ0QGhwGoihiATJg6D7h3SBt47v0FSPm657i21gaeOs/W3K0ic/PgqFh7K2m3+N7KVYSgpmOrd8TYkfeA88vENbvFAu4h1XdtIjJoDEBVxABkQiRJjExKfHRLK3kvkJ+ue45DfdFHxLMb4NVVtBpQzZAk4M4OIH4SkHJKHLP3A1p9DPgPBpQWVbt+wjIx3w8AdNoM1HutatcjIqPCAFRFDEBGTpKAG/8Vq30nxYh+PUXZ1AY8u4oWHs9uYsg3Rwbpl7oAuPoNcHK66KwMAK6tREdp7/DK/T5u/Qbs6yP6TgXNB56aXL1lJiKDxwBURQxARu6fpcCx8drvLewAj87awFOrtfxz05CQnyVuV52ZC+SlimOe3YA2CwG3p5/8Og/jgV2dgPwMoOEIoP0XDLVEZqgin9+yfwqsWLEC/v7+sLW1RUhICA4fPlzmuWfOnEG/fv3g7+8PhUKBJUuWVPmaZGIenhC3VwAx70u3WOC1h8Dz24HmE8QwbIYfw2FpBzw1CehzCWgWJTpIJ8UA29sCBwaLDtSPk3kLiH1JhB/Pbpzrh4ieiKyfBJs2bUJUVBRmzpyJY8eOoXXr1ggPD0dycnKp52dmZqJBgwaYP38+vLy8quWaZELyM4EDAwF1LuDbR3wQenYBLGzkLhk9jk1t4OnFwEsJoi8QIEaNbWsGHI0qOf9Sobx0YG9vMfGic3Pg2f9ylB4RPRFZb4GFhISgXbt2WL58OQBArVbDz88PY8eOxZQpU8p9rr+/P8aPH4/x48dX2zUL8RaYkTo8Gri4CrDzBnqeBGzd5S4RVdaDY2JpjcQ/xPdWLsBTU4Cm40SrESD6Ef35CnDrV8CmDhB+SMxiTURmyyhugeXm5uLo0aMICwvTFkapRFhYGOLi4vR6zZycHKhUKp2NjMyNrSL8QAGEfs3wY+zcnga67gKe3yGmJchLBU5MBbY1AS6vE+Hn+AQRfpQ2QOefGX6IqEJkC0D37t1DQUEBPD09dY57enoiMTFRr9ecN28eXFxcNJufn1+lXp9kknkLODRC7DefKObwIdPg3R3oeQwI3SAWWM28Cfw1HPi1IZCwRJwTugGoEyprMYnI+LA3KICpU6ciNTVVs924cUPuItGTUhcAcUPE8hBubYFWs+UuEVU3hRIIGAL0TgDafCImUMy4Jh5rPQeoP0DW4hGRcbKU64Xd3d1hYWGBpKQkneNJSUlldnCuqWva2NjAxoYdZY3SP4uApD2ApQPQcSNgYS13iaimWNiKkXwN3hRD5y2dgGbvy10qIjJSsrUAWVtbo23btoiJidEcU6vViImJQWho5Zqza+KaZMDuHRZrTAFA22ViAVAyfTZuQOBMoHkUh7sTUaXJ1gIEAFFRUYiMjERwcDDat2+PJUuWICMjA8OHDwcADB06FL6+vpg3bx4A0cn57Nmzmv1bt24hPj4ejo6OaNSo0RNdk0xEXhpwcJBYx6veAKDBMLlLRERERkTWABQREYG7d+8iOjoaiYmJCAoKwvbt2zWdmK9fvw6lUttIdfv2bbRp00bz/aJFi7Bo0SJ06dIFsbGxT3RNMhFHxgLpl0TH2Par2BJAREQVwqUwSsF5gAzc1e9E649CKWZ69nhW7hIREZEBMIp5gIgqJf0q8Pcosd9iOsMPERFVCgMQGQ91PnBwMJCnAtxDgZYz5C4REREZKQYgMh6nPwbuHQSsnIGO3wJKWbuwERGREWMAIuOQvB8482iSw3aruOwBERFVCQMQGb7cFHHrS1IDAUMB/4Fyl4iIiIwcAxAZNkkCDr8NZF4HHBsAwcvlLhEREZkABiAybFfWA9d/ABSWQMfvACsnuUtEREQmgAGIDJfqAnBkjNhv9W/Avb285SEiIpPBAESGqSBXTHaYnwF4PAc0nyR3iYiIyIQwAJFhOhUNPDgCWNcCOn4NKC3kLhEREZkQBiAyPIkxwNmFYj/kP4B9XXnLQ0REJocBiAxL9j0gbigACWj0FuD3qtwlIiIiE8QARIZDkoDD/wKybgPOzYCnP5W7REREZKIYgMhwXFwN3PwZUFoDHTcClg5yl4iIiEwUAxAZhtSzwLH3xX7QfMCtjbzlISIik8YARPIryAYODBRfvcOBpuPkLhEREZk4BiCSX/wUIOUkYFMH6LAOUPCfJRER1Sx+0pC8bv0OJCwV+x3WAXZeshaHiIjMAwMQyScrCTg0XOw3eQ/wfVHe8hARkdlgACJ5SGrgr2FAdjLgGgi0WSB3iYiIyIwwAJE8Ej4H7mwHLGzFKu8WtnKXiIiIzAgDEOnfw3ggfrLYf/pTwLWFrMUhIiLzwwBE+pWfKYa8q3OBui8DjUbJXSIiIjJDDECkX8eiANU/gJ0P0P4/gEIhd4mIiMgMMQCR/tz4SSx3AQUQugGwdZe7REREZKYYgEg/Mm8Ch/4l9p+aBHh1k7c8RERk1hiAqOapC4CDQ4DcB4BbMBD4b7lLREREZo4BiGreuYVAcqxY3b3jRsDCWu4SERGRmWMAopp17xBwcobYD14OODeWtzxEREQALOUuAJkYSQKybouRXqp/gHOLAakAqBcBBETKXToiIiIADEBUWQW5QPpFbdBJPafdz0/XPde+HtB+FYe8ExGRwWAAovLlPgRS/9GGG9U/gOockH5ZtOyURmEBODYEnJsBLs3FZIfWrnotNhERUXkYgEgsTJpxvVjIeRR0spPLfp6lkwg5zs0Al0dfnZuL8MOOzkREZMAYgPQp96HYoAQUjzbNvkXZx4qer6hCv/X8LCDtfClBJwEoyCr7efZ1tUHHuUjQsfPmbS0iIjJKDED6dGE1cGJq1a9TIhRZoERIKn5MUgNZdwBIpV9TaQ04NdYNOS7NAacmgJVT1ctMRERkQBiA9ElpBVg6ijACtfgqqR/1pSkjmJRG8/xKlMG6lmi9KRpynJsBDv6Akv8ciIjIPPATT5+afyC20kgSAKlYKCoSkqAWMyoXP1Z4blnHCr+HJEZj2bjzthUREZk9BiBDoVAAUFStjw8RERE9EX7aEhERkdlhACIiIiKzwwBEREREZocBiIiIiMwOAxARERGZHQYgIiIiMjsMQERERGR2GICIiIjI7DAAERERkdlhACIiIiKzwwBEREREZocBiIiIiMwOAxARERGZHQYgIiIiMjsMQERERGR2DCIArVixAv7+/rC1tUVISAgOHz5c7vmbN29Gs2bNYGtri8DAQPz+++86jw8bNgwKhUJn69GjR01WgYiIiIyI7AFo06ZNiIqKwsyZM3Hs2DG0bt0a4eHhSE5OLvX8gwcPYuDAgRgxYgSOHz+Ovn37om/fvjh9+rTOeT169MCdO3c023fffaeP6hAREZERUEiSJMlZgJCQELRr1w7Lly8HAKjVavj5+WHs2LGYMmVKifMjIiKQkZGBbdu2aY516NABQUFBWLVqFQDRApSSkoKtW7c+URlycnKQk5Oj+V6lUsHPzw+pqalwdnauQu2IiIhIX1QqFVxcXJ7o81vWFqDc3FwcPXoUYWFhmmNKpRJhYWGIi4sr9TlxcXE65wNAeHh4ifNjY2Ph4eGBpk2bYvTo0bh//36Z5Zg3bx5cXFw0m5+fXxVqRURERIZO1gB07949FBQUwNPTU+e4p6cnEhMTS31OYmLiY8/v0aMHNmzYgJiYGCxYsAB79+5Fz549UVBQUOo1p06ditTUVM1248aNKtaMiIiIDJml3AWoCa+//rpmPzAwEK1atULDhg0RGxuLbt26lTjfxsYGNjY2+iwiERERyUjWFiB3d3dYWFggKSlJ53hSUhK8vLxKfY6Xl1eFzgeABg0awN3dHRcvXqx6oYmIiMjoyRqArK2t0bZtW8TExGiOqdVqxMTEIDQ0tNTnhIaG6pwPALt27SrzfAC4efMm7t+/D29v7+opOBERERk12YfBR0VF4YsvvsD69etx7tw5jB49GhkZGRg+fDgAYOjQoZg6darm/HHjxmH79u1YvHgx/vnnH8yaNQtHjhzBmDFjAADp6emYOHEi/vrrL1y9ehUxMTF4+eWX0ahRI4SHh8tSRyIiIjIssvcBioiIwN27dxEdHY3ExEQEBQVh+/btmo7O169fh1KpzWkdO3bExo0bMX36dHz44Ydo3Lgxtm7dipYtWwIALCwscPLkSaxfvx4pKSnw8fFB9+7dMXv2bPbzISIiIgAGMA+QIarIPAJERERkGIxmHiAiIiIiOTAAERERkdlhACIiIiKzwwBEREREZocBiIiIiMwOAxARERGZHQYgIiIiMjsMQERERGR2GICIiIjI7DAAERERkdlhACIiIiKzwwBEREREZocBiIiIiMwOAxARERGZHQYgIiIiMjsMQERERGR2GICIiIjI7DAAERERkdlhACIiIiKzwwBEREREZocBiIiIiMwOAxARERGZHQYgIiIiMjsMQERERGR2GICIiIjI7DAAERERkdlhACIiIiKzwwBEREREZocBiIiIiMwOAxARERGZHQYgIiIiMjsMQERERGR2GICIiIjI7DAAERERkdlhACIiIiKzwwBEREREZocBiIiIiMwOAxARERGZHQYgIiIiMjsMQERERGR2GICIiIjI7DAAERERkdlhACIiIiKzwwBEREREZocBiIiIiMwOAxARERGZHQYgIiIiMjsMQERERGR2GICIiIjI7DAAERERkdlhACIiIiKzwwBEREREZscgAtCKFSvg7+8PW1tbhISE4PDhw+Wev3nzZjRr1gy2trYIDAzE77//rvO4JEmIjo6Gt7c37OzsEBYWhgsXLtRkFYiIiMiIyB6ANm3ahKioKMycORPHjh1D69atER4ejuTk5FLPP3jwIAYOHIgRI0bg+PHj6Nu3L/r27YvTp09rzlm4cCE+//xzrFq1CocOHYKDgwPCw8ORnZ2tr2oRERGRAVNIkiTJWYCQkBC0a9cOy5cvBwCo1Wr4+flh7NixmDJlSonzIyIikJGRgW3btmmOdejQAUFBQVi1ahUkSYKPjw8++OADTJgwAQCQmpoKT09PrFu3Dq+//vpjy6RSqeDi4oLU1FQ4OztXU01Fy1RmXma1XY+IiMhY2VvZQ6FQVOs1K/L5bVmtr1xBubm5OHr0KKZOnao5plQqERYWhri4uFKfExcXh6ioKJ1j4eHh2Lp1KwDgypUrSExMRFhYmOZxFxcXhISEIC4urtQAlJOTg5ycHM33KpWqKtUqU2ZeJhznOdbItYmIiIxJ+tR0OFg7yPb6st4Cu3fvHgoKCuDp6alz3NPTE4mJiaU+JzExsdzzC79W5Jrz5s2Di4uLZvPz86tUfYiIiMg4yNoCZCimTp2q06qkUqlqJATZW9kjfWp6tV+XiIjI2Nhb2cv6+rIGIHd3d1hYWCApKUnneFJSEry8vEp9jpeXV7nnF35NSkqCt7e3zjlBQUGlXtPGxgY2NjaVrcYTUygUsjb3ERERkSDrLTBra2u0bdsWMTExmmNqtRoxMTEIDQ0t9TmhoaE65wPArl27NOcHBATAy8tL5xyVSoVDhw6VeU0iIiIyL7LfAouKikJkZCSCg4PRvn17LFmyBBkZGRg+fDgAYOjQofD19cW8efMAAOPGjUOXLl2wePFi9OrVC99//z2OHDmCNWvWABCtLOPHj8fHH3+Mxo0bIyAgADNmzICPjw/69u0rVzWJiIjIgMgegCIiInD37l1ER0cjMTERQUFB2L59u6YT8/Xr16FUahuqOnbsiI0bN2L69On48MMP0bhxY2zduhUtW7bUnDNp0iRkZGTgrbfeQkpKCjp16oTt27fD1tZW7/UjIiIiwyP7PECGqKbmASIiIqKaU5HPb9lngiYiIiLSNwYgIiIiMjsMQERERGR2GICIiIjI7DAAERERkdlhACIiIiKzwwBEREREZocBiIiIiMwOAxARERGZHdmXwjBEhZNjq1QqmUtCRERET6rwc/tJFrlgACpFWloaAMDPz0/mkhAREVFFpaWlwcXFpdxzuBZYKdRqNW7fvg0nJycoFAq5i1MlKpUKfn5+uHHjhsmva8a6mh5zqSfAupoic6knYDh1lSQJaWlp8PHx0VlIvTRsASqFUqlE3bp15S5GtXJ2djb5/4CFWFfTYy71BFhXU2Qu9QQMo66Pa/kpxE7QREREZHYYgIiIiMjsMACZOBsbG8ycORM2NjZyF6XGsa6mx1zqCbCupshc6gkYZ13ZCZqIiIjMDluAiIiIyOwwABEREZHZYQAiIiIis8MARERERGaHAcgIzJs3D+3atYOTkxM8PDzQt29fJCQk6JyTnZ2Nd999F7Vr14ajoyP69euHpKQknXOuX7+OXr16wd7eHh4eHpg4cSLy8/N1zomNjcXTTz8NGxsbNGrUCOvWravp6pVp/vz5UCgUGD9+vOaYKdXz1q1beOONN1C7dm3Y2dkhMDAQR44c0TwuSRKio6Ph7e0NOzs7hIWF4cKFCzrXePDgAQYPHgxnZ2e4urpixIgRSE9P1znn5MmTePbZZ2Fraws/Pz8sXLhQL/UrVFBQgBkzZiAgIAB2dnZo2LAhZs+erbNWj7HWdd++fejduzd8fHygUCiwdetWncf1Wa/NmzejWbNmsLW1RWBgIH7//Xe91DMvLw+TJ09GYGAgHBwc4OPjg6FDh+L27dtGV8/H1bW4UaNGQaFQYMmSJTrHTamu586dQ58+feDi4gIHBwe0a9cO169f1zxu1O/JEhm88PBwae3atdLp06el+Ph46cUXX5Tq1asnpaena84ZNWqU5OfnJ8XExEhHjhyROnToIHXs2FHzeH5+vtSyZUspLCxMOn78uPT7779L7u7u0tSpUzXnXL58WbK3t5eioqKks2fPSsuWLZMsLCyk7du367W+kiRJhw8flvz9/aVWrVpJ48aN0xw3lXo+ePBAql+/vjRs2DDp0KFD0uXLl6UdO3ZIFy9e1Jwzf/58ycXFRdq6dat04sQJqU+fPlJAQICUlZWlOadHjx5S69atpb/++kv6888/pUaNGkkDBw7UPJ6amip5enpKgwcPlk6fPi199913kp2dnbR69Wq91XXOnDlS7dq1pW3btklXrlyRNm/eLDk6OkpLly41+rr+/vvv0rRp06QtW7ZIAKSffvpJ53F91evAgQOShYWFtHDhQuns2bPS9OnTJSsrK+nUqVM1Xs+UlBQpLCxM2rRpk/TPP/9IcXFxUvv27aW2bdvqXMMY6vm4uha1ZcsWqXXr1pKPj4/02WefmWRdL168KLm5uUkTJ06Ujh07Jl28eFH6+eefpaSkJM05xvyezABkhJKTkyUA0t69eyVJEm9AVlZW0ubNmzXnnDt3TgIgxcXFSZIk/qErlUopMTFRc87KlSslZ2dnKScnR5IkSZo0aZLUokULndeKiIiQwsPDa7pKOtLS0qTGjRtLu3btkrp06aIJQKZUz8mTJ0udOnUq83G1Wi15eXlJn3zyieZYSkqKZGNjI3333XeSJEnS2bNnJQDS33//rTnnf//7n6RQKKRbt25JkiRJ//d//yfVqlVLU/fC127atGl1V6lMvXr1kt58802dY6+++qo0ePBgSZJMp67FP0D0Wa8BAwZIvXr10ilPSEiI9Pbbb1drHSWpZD1Lc/jwYQmAdO3aNUmSjLOeklR2XW/evCn5+vpKp0+flurXr68TgEyprhEREdIbb7xR5nOM/T2Zt8CMUGpqKgDAzc0NAHD06FHk5eUhLCxMc06zZs1Qr149xMXFAQDi4uIQGBgIT09PzTnh4eFQqVQ4c+aM5pyi1yg8p/Aa+vLuu++iV69eJcpiSvX85ZdfEBwcjP79+8PDwwNt2rTBF198oXn8ypUrSExM1Cmni4sLQkJCdOrq6uqK4OBgzTlhYWFQKpU4dOiQ5pzOnTvD2tpac054eDgSEhLw8OHDmq4mAKBjx46IiYnB+fPnAQAnTpzA/v370bNnTwCmVdei9FkvQ/g3XVRqaioUCgVcXV0BmFY91Wo1hgwZgokTJ6JFixYlHjeVuqrVavz2229o0qQJwsPD4eHhgZCQEJ3bZMb+nswAZGTUajXGjx+PZ555Bi1btgQAJCYmwtraWvNmU8jT0xOJiYmac4r+Ayx8vPCx8s5RqVTIysqqieqU8P333+PYsWOYN29eicdMqZ6XL1/GypUr0bhxY+zYsQOjR4/Ge++9h/Xr1+uUtbRyFq2Hh4eHzuOWlpZwc3Or0M+jpk2ZMgWvv/46mjVrBisrK7Rp0wbjx4/H4MGDdcphCnUtSp/1KuscOeqdnZ2NyZMnY+DAgZpFMU2pngsWLIClpSXee++9Uh83lbomJycjPT0d8+fPR48ePbBz50688sorePXVV7F3715NGY35PZmrwRuZd999F6dPn8b+/fvlLkq1u3HjBsaNG4ddu3bB1tZW7uLUKLVajeDgYMydOxcA0KZNG5w+fRqrVq1CZGSkzKWrXj/88AO+/fZbbNy4ES1atEB8fDzGjx8PHx8fk6urucvLy8OAAQMgSRJWrlwpd3Gq3dGjR7F06VIcO3YMCoVC7uLUKLVaDQB4+eWX8f777wMAgoKCcPDgQaxatQpdunSRs3jVgi1ARmTMmDHYtm0b9uzZg7p162qOe3l5ITc3FykpKTrnJyUlwcvLS3NO8Z75hd8/7hxnZ2fY2dlVd3VKOHr0KJKTk/H000/D0tISlpaW2Lt3Lz7//HNYWlrC09PTJOoJAN7e3njqqad0jjVv3lwzuqKwrKWVs2g9kpOTdR7Pz8/HgwcPKvTzqGkTJ07UtAIFBgZiyJAheP/99zWtfKZU16L0Wa+yztFnvQvDz7Vr17Br1y5N609h+Uyhnn/++SeSk5NRr149zXvUtWvX8MEHH8Df319TRlOoq7u7OywtLR/7PmXM78kMQEZAkiSMGTMGP/30E3bv3o2AgACdx9u2bQsrKyvExMRojiUkJOD69esIDQ0FAISGhuLUqVM6/zEL36QK/4GHhobqXKPwnMJr1LRu3brh1KlTiI+P12zBwcEYPHiwZt8U6gkAzzzzTImpDM6fP4/69esDAAICAuDl5aVTTpVKhUOHDunUNSUlBUePHtWcs3v3bqjVaoSEhGjO2bdvH/Ly8jTn7Nq1C02bNkWtWrVqrH5FZWZmQqnUfauxsLDQ/IVpSnUtSp/1kvvfdGH4uXDhAv744w/Url1b53FTqeeQIUNw8uRJnfcoHx8fTJw4ETt27NCU0RTqam1tjXbt2pX7PmX0nz012sWaqsXo0aMlFxcXKTY2Vrpz545my8zM1JwzatQoqV69etLu3bulI0eOSKGhoVJoaKjm8cKhiN27d5fi4+Ol7du3S3Xq1Cl1KOLEiROlc+fOSStWrJBtGHyhoqPAJMl06nn48GHJ0tJSmjNnjnThwgXp22+/lezt7aVvvvlGc878+fMlV1dX6eeff5ZOnjwpvfzyy6UOoW7Tpo106NAhaf/+/VLjxo11htumpKRInp6e0pAhQ6TTp09L33//vWRvb6/XYfCRkZGSr6+vZhj8li1bJHd3d2nSpElGX9e0tDTp+PHj0vHjxyUA0qeffiodP35cM/pJX/U6cOCAZGlpKS1atEg6d+6cNHPmzGodMl1ePXNzc6U+ffpIdevWleLj43Xeo4qOcjKGej6urqUpPgrMlOq6ZcsWycrKSlqzZo104cIFzfD0P//8U3MNY35PZgAyAgBK3dauXas5JysrS3rnnXekWrVqSfb29tIrr7wi3blzR+c6V69elXr27CnZ2dlJ7u7u0gcffCDl5eXpnLNnzx4pKChIsra2lho0aKDzGnIoHoBMqZ6//vqr1LJlS8nGxkZq1qyZtGbNGp3H1Wq1NGPGDMnT01OysbGRunXrJiUkJOicc//+fWngwIGSo6Oj5OzsLA0fPlxKS0vTOefEiRNSp06dJBsbG8nX11eaP39+jdetKJVKJY0bN06qV6+eZGtrKzVo0ECaNm2azoejsdZ1z549pf7fjIyM1Hu9fvjhB6lJkyaStbW11KJFC+m3337TSz2vXLlS5nvUnj17jKqej6traUoLQKZU1y+//FJq1KiRZGtrK7Vu3VraunWrzjWM+T1ZIUlFpmMlIiIiMgPsA0RERERmhwGIiIiIzA4DEBEREZkdBiAiIiIyOwxAREREZHYYgIiIiMjsMAARERGR2WEAIiIiIrPDAEREJuu5557D+PHjAQD+/v5YsmSJrOUhIsNhKXcBiIj04e+//4aDg4PcxSAiA8EARERmoU6dOnIXgYgMCG+BEZFJyMjIwNChQ+Ho6Ahvb28sXrxY5/Hit8AUCgVWr16Nl156Cfb29mjevDni4uJw8eJFPPfcc3BwcEDHjh1x6dIlPdeEiPSBAYiITMLEiROxd+9e/Pzzz9i5cydiY2Nx7Nixcp8ze/ZsDB06FPHx8WjWrBkGDRqEt99+G1OnTsWRI0cgSRLGjBmjpxoQkT7xFhgRGb309HR8+eWX+Oabb9CtWzcAwPr161G3bt1ynzd8+HAMGDAAADB58mSEhoZixowZCA8PBwCMGzcOw4cPr9nCE5Es2AJEREbv0qVLyM3NRUhIiOaYm5sbmjZtWu7zWrVqpdn39PQEAAQGBuocy87OhkqlquYSE5HcGICIyGxZWVlp9hUKRZnH1Gq1fgtGRDWOAYiIjF7Dhg1hZWWFQ4cOaY49fPgQ58+fl7FURGTI2AeIiIyeo6MjRowYgYkTJ6J27drw8PDAtGnToFTybzwiKh0DEBGZhE8++QTp6eno3bs3nJyc8MEHHyA1NVXuYhGRgVJIkiTJXQgiIiIifWL7MBEREZkdBiAiIiIyOwxAREREZHYYgIiIiMjsMAARERGR2WEAIiIiIrPDAERERERmhwGIiIiIzA4DEBEREZkdBiAiIiIyOwxAREREZHb+H6wWMKFKNAfPAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "backward:\n",
      "        dim  Triton-DYT  Torch-DYT   RMSNorm  Liger-DYT\n",
      "0    1024.0    0.142407        0.0  0.066771   0.139540\n",
      "1    2048.0    0.136292        0.0  0.066570   0.138657\n",
      "2    3072.0    0.137973        0.0  0.070274   0.139825\n",
      "3    4096.0    0.141268        0.0  0.103122   0.181623\n",
      "4    5120.0    0.175215        0.0  0.109353   0.181678\n",
      "5    6144.0    0.175732        0.0  0.107766   0.192801\n",
      "6    7168.0    0.177004        0.0  0.111275   0.190093\n",
      "7    8192.0    0.175612        0.0  0.098411   0.160719\n",
      "8    9216.0    0.158014        0.0  0.126017   0.168756\n",
      "9   10240.0    0.163639        0.0  0.129156   0.213680\n",
      "10  11264.0    0.169940        0.0  0.133572   0.224484\n",
      "11  12288.0    0.174275        0.0  0.136550   0.229402\n",
      "12  13312.0    0.175741        0.0  0.141127   0.239472\n",
      "13  14336.0    0.181681        0.0  0.144843   0.247482\n",
      "14  15360.0    0.185152        0.0  0.147946   0.256967\n",
      "15  16384.0    0.189480        0.0  0.150967   0.263907\n"
     ]
    }
   ],
   "source": [
    "\n",
    "@triton.testing.perf_report(\n",
    "    triton.testing.Benchmark(\n",
    "        x_names=['dim'],  # argument names to use as an x-axis for the plot\n",
    "        x_vals=[1024 * i for i in range(1, 16+1)],  # different possible values for `x_name`\n",
    "        line_arg='provider',  # argument name whose value corresponds to a different line in the plot\n",
    "        line_vals=['Triton-DYT', \"Torch-DYT\", 'RMSNorm', \"Liger-DYT\"],  # possible values for `line_arg``\n",
    "        line_names=[\n",
    "            \"Triton-DYT\",\n",
    "            \"Torch-DYT\",\n",
    "            \"RMSNorm\",\n",
    "            \"Liger-DYT\"\n",
    "        ],  # label name for the lines\n",
    "        styles=[('blue', '-'), ('green', '-'), ('orange', '-'), ('red', '-')],  # line styles\n",
    "        ylabel=\"ms\",  # label name for the y-axis\n",
    "        plot_name=\"backward\",  # name for the plot. Used also as a file name for saving the plot.\n",
    "        args={'seq_len': 1024, 'bs': 4, 'beta':True}\n",
    "    ))\n",
    "def benchmark(bs, seq_len, dim, beta, provider):\n",
    "    device = torch.device('cuda')\n",
    "    dtype = torch.bfloat16\n",
    "    tensor = torch.randn(bs, seq_len, dim).to(device).to(dtype)\n",
    "    tensor.requires_grad_(True)\n",
    "    dy = torch.rand_like(tensor)\n",
    "    stream = torch.cuda.Stream()\n",
    "    torch.cuda.set_stream(stream)\n",
    "    if provider == 'Triton-DYT':\n",
    "        dyt = DYT(dim, beta=beta).to(device).to(dtype)\n",
    "        y = dyt(tensor, 'triton')\n",
    "        ms = triton.testing.do_bench(lambda: y.backward(dy, retain_graph=True), grad_to_none=[tensor])\n",
    "    if provider == 'Torch-DYT':\n",
    "        return 0\n",
    "        dyt = DYT(dim, beta=beta).to(device).to(dtype)\n",
    "        y = dyt(tensor, 'torch')\n",
    "        ms = triton.testing.do_bench(lambda: y.backward(dy, retain_graph=True), grad_to_none=[tensor])\n",
    "    if provider == 'RMSNorm':\n",
    "        norm = TritonRMSNorm(dim).to(device).to(dtype)\n",
    "        # norm = FusedRMSNorm(dim).to(device).to(dtype)\n",
    "        # norm = TERMSNorm(dim).to(device).to(dtype)\n",
    "        y = norm(tensor)\n",
    "        ms = triton.testing.do_bench(lambda: y.backward(dy, retain_graph=True), grad_to_none=[tensor])\n",
    "    if provider == 'Liger-DYT':\n",
    "        dyt = DYT2(dim, beta=beta).to(device).to(dtype)\n",
    "        y = dyt(tensor, 'triton')\n",
    "        ms = triton.testing.do_bench(lambda: y.backward(dy, retain_graph=True), grad_to_none=[tensor])\n",
    "        # if not beta:\n",
    "        #     return 0\n",
    "        # dyt = LigerDyT(dim).to(device).to(dtype)\n",
    "        # y = dyt(tensor)\n",
    "        # ms = triton.testing.do_bench(lambda: y.backward(dy, retain_graph=True), grad_to_none=[tensor])\n",
    "    return ms\n",
    "benchmark.run(show_plots=True, print_data=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
